From b5eb1485a181a0e9afaf50f596ba2da506b021b6 Mon Sep 17 00:00:00 2001 From: Thane Somers Date: Mon, 27 Feb 2017 18:02:34 -0800 Subject: [PATCH 01/38] Add Absolute Ratings This isn't working now and the code's a mess from debugging, but it's a start? --- GP_preference_demo.py | 62 +++++++++++----- GPpref.py | 162 ++++++++++++++++++++++++++++++++++++++---- 2 files changed, 193 insertions(+), 31 deletions(-) diff --git a/GP_preference_demo.py b/GP_preference_demo.py index ee18e19..f782fd6 100644 --- a/GP_preference_demo.py +++ b/GP_preference_demo.py @@ -4,16 +4,18 @@ import GPpref import scipy.optimize as op -log_hyp = np.log([0.1,1.0,0.1]) # length_scale, sigma_f, sigma_probit -np.random.seed(1) +log_hyp = np.log([0.1,1.0,0.1,10.0]) # length_scale, sigma_f, sigma_probit, v_beta +np.random.seed(2) -n_train = 20 +n_rel_train = 0 +n_abs_train = 5 true_sigma = 0.05 delta_f = 1e-5 # Define polynomial function to be modelled def true_function(x): - y = np.sin(x*2*np.pi + np.pi/4) + 0.2 + y = (np.sin(x*2*np.pi + np.pi/4) + 1)/2 + #y = np.sin(x*2.0*np.pi + np.pi/4.0) return y # Define noisy observation function @@ -32,39 +34,63 @@ def noisy_preference_rank(uv, sigma): # Plot true function x_plot = np.linspace(0.0,1.0,101,dtype='float') y_plot = true_function(x_plot) +hf,ha = plt.subplots(1,1) +ha.plot(x_plot,y_plot,'r-') + # Training data - this is a bit weird, but we sample x values, then the uv pairs # are actually indexes into x, because it is easier computationally. You can # recover the actual u,v values using x[ui],x[vi] -x_train = np.random.random((2*n_train,1)) -uvi_train = np.random.choice(range(2*n_train), (n_train,2), replace=False) -uv_train = x_train[uvi_train][:,:,0] +if n_rel_train > 0: + x_train = np.random.random((2*n_rel_train,1)) + uvi_train = np.random.choice(range(2*n_rel_train), (n_rel_train,2), replace=False) + uv_train = x_train[uvi_train][:,:,0] -# Get noisy observations f(uv) and corresponding ranks y_train -y_train, fuv_train = noisy_preference_rank(uv_train,true_sigma) + # Get noisy observations f(uv) and corresponding ranks y_train + y_train, fuv_train = noisy_preference_rank(uv_train, true_sigma) + for uv,fuv,y in zip(uv_train, fuv_train, y_train): + ha.plot(uv, fuv, 'b-') + ha.plot(uv[(y+1)/2],fuv[(y+1)/2],'k+') -hf,ha = plt.subplots(1,1) -ha.plot(x_plot,y_plot,'r-') -for uv,fuv,y in zip(uv_train, fuv_train, y_train): - ha.plot(uv, fuv, 'b-') - ha.plot(uv[(y+1)/2],fuv[(y+1)/2],'k+') +else: + x_train = np.zeros((0,1)) + uvi_train = np.zeros((0,2)) + uv_train = np.zeros((0,2)) + y_train = np.zeros((0,1)) + fuv_train = np.zeros((0,2)) + +x_abs_train = np.random.random((n_abs_train,1)) +#y_abs_train = obs_function(x_abs_train, true_sigma) +y_abs_train = np.clip(obs_function(x_abs_train, true_sigma), 0.01, .99) + +ha.plot(x_abs_train, y_abs_train, 'r+') ha.set_title('Training data') ha.set_ylabel('f(x)') ha.set_xlabel('x') - # GP hyperparameters # note: using log scaling to aid learning hyperparameters with varied magnitudes -prefGP = GPpref.PreferenceGaussianProcess(x_train, uvi_train, y_train, delta_f=delta_f) +print "Data" +# print x_train +print x_abs_train +# print y_train +print y_abs_train +# print uvi_train + +prefGP = GPpref.PreferenceGaussianProcess(x_train, uvi_train, x_abs_train, y_train, y_abs_train, delta_f=delta_f) # Pseudocode: # FOr a set of hyperparameters, return log likelihood that can be used by an optimiser theta0 = log_hyp # log_hyp = op.fmin(prefGP.calc_nlml,theta0) -f,lml = prefGP.calc_laplace(log_hyp) +#f,lml = prefGP.calc_laplace(log_hyp) +f = prefGP.calc_laplace(log_hyp) -ha.plot(x_train, f, 'g^') +if x_train.shape[0]>0: + ha.plot(x_train, f[0:x_train.shape[0]], 'g^') +if x_abs_train.shape[0]>0: + ha.plot(x_abs_train, f[x_train.shape[0]:], 'r^') plt.show() diff --git a/GPpref.py b/GPpref.py index f19ccd2..e15cc8d 100644 --- a/GPpref.py +++ b/GPpref.py @@ -1,13 +1,21 @@ # Simple 1D GP classification example import numpy as np import GPy -from scipy.special import ndtr as std_norm_cdf +#from scipy.special import ndtr as std_norm_cdf, digamma, polygamma +from scipy.special import digamma, polygamma +from scipy.stats import norm +from scipy.linalg import block_diag #define a standard normal pdf _sqrt_2pi = np.sqrt(2*np.pi) def std_norm_pdf(x): x = np.clip(x,-1e150,1e150) - return np.exp(-(x**2)/2)/_sqrt_2pi + return norm.pdf(x) + #return np.exp(-(x**2)/2)/_sqrt_2pi + +def std_norm_cdf(x): + x = np.clip(x, -30, 100 ) + return norm.cdf(x) # Define squared distance calculation function def squared_distance(A,B): @@ -85,6 +93,7 @@ def derivatives(self, uvi, y, f): W[yi, yi] -= ddpy_df # If x_i = x_i = v_k then I(x_i)*I(y_i) = -1*-1 = 1 W[xi, yi] -= -ddpy_df # Otherwise, I(x_i)*I(y_i) = -1*1 = -1 W[yi, xi] -= -ddpy_df + return W, dpy_df def log_marginal(self, uvi, y, f, iK, logdetK): @@ -93,18 +102,116 @@ def log_marginal(self, uvi, y, f, iK, logdetK): psi = np.sum(np.log(phi_z)) - 0.5 * np.matmul(np.matmul(f.T, iK), f) - 0.5*logdetK - iK.shape[0]/2.0*self.log2pi return psi.flat[0] +class AbsBoundProbit(object): + def __init__(self, sigma=1.0, v=10.0): + # Making v=10 looks good on a graph, but I'm not sure what it's actually supposed to be. + self.set_sigma(sigma) + self.set_v(v) + self.log2pi = np.log(2.0*np.pi) + + def set_sigma(self, sigma): + self.sigma = sigma + self._isqrt2sig = 1.0 / (self.sigma * np.sqrt(2.0)) + self._i2var = self._isqrt2sig**2 + + def set_v(self, v): + self.v = v + + def mean_link(self, f): + return std_norm_cdf(f*self._isqrt2sig) + + def alpha(self, f): + return self.v * self.mean_link(f) + + def beta(self, f): + return self.v * (1-self.mean_link(f)) + + def derivatives(self, y, f): + + print "Start Iter. f, y" + print f + print y + alpha = self.alpha(f) + beta = self.beta(f) #let's make a distribution called beta that also has beta as a parameter! + # print "Alpha: " + str(alpha) + # print "Beta: " + str(beta) + + # Theres a dot in Jensen that I'm hoping is just a typo. I didn't fully derive it, but it looks correct. + # As the iteration goes, f blows up. This makes parts of alpha, beta go to v and 0. Digamma(0)=-inf :( + # So why is it blowing up? + dpy_df = self.v*std_norm_pdf(f*self._isqrt2sig) * (np.log(y)-np.log(1-y) - digamma(alpha) + digamma(beta) ) + + Wdiag = ( - self.v**2*std_norm_pdf(f*self._isqrt2sig) * + ( std_norm_pdf(f*self._isqrt2sig) * ( polygamma(0, alpha) + polygamma(0, alpha) ) + + f*self._i2var/self.v * (np.log(y)-np.log(1-y)-digamma(alpha) + digamma(beta)) )) + + W = np.diagflat(Wdiag) + + return W, dpy_df + + def log_marginal(self): + pass + + +class AbsProbit(object): + # The Probit Likelihood given in Rasmussen, p43. Note f(x) is scaled by sqrt(2)*sigma, as in Jensen. + # Ok, yeah, this was totally wrong. Don't use this. + def __init__(self, sigma=1.0, v=10.0): + self.set_sigma(sigma) + self.log2pi = np.log(2.0*np.pi) + + def set_sigma(self, sigma): + self.sigma = sigma + self._isqrt2sig = 1.0 / (self.sigma * np.sqrt(2.0)) + self._i2var = self._isqrt2sig**2 + + def derivatives(self, y, f): + + print "Start Iter. f, y" + print f + print y + + # dpy_df = -y*std_norm_pdf(f*self._isqrt2sig) / std_norm_cdf(y*f*self._isqrt2sig) + + # Wdiag = -(-np.power(std_norm_pdf(f*self._isqrt2sig) / std_norm_cdf(y*f*self._isqrt2sig),2) + # -y*f*self._isqrt2sig*std_norm_pdf(f*self._isqrt2sig) / std_norm_cdf(y*f*self._isqrt2sig)) + + dpy_df = -y*std_norm_pdf(f) / std_norm_cdf(y*f) + + Wdiag = -(-np.power(std_norm_pdf(f) / std_norm_cdf(y*f),2) + -y*f*std_norm_pdf(f) / std_norm_cdf(y*f)) + + W = np.diagflat(Wdiag) + + return W, dpy_df + + + class PreferenceGaussianProcess(object): - def __init__(self, x_train, uvi_train, y_train, likelihood=PrefProbit, delta_f = 1e-6): - # log_hyp are log of hyperparameters, note that it is [length_0, ..., length_d, sigma_f, sigma_probit] - self._xdim = x_train.shape[1] - self._nx = x_train.shape[0] + def __init__(self, x_train, uvi_train, x_abs_train, y_train, y_abs_train, likelihood=PrefProbit, delta_f = 1e-6): + # log_hyp are log of hyperparameters, note that it is [length_0, ..., length_d, sigma_f, sigma_probit, v_beta] + # Training points are split into relative and absolute for calculating f, but combined for predictions. + if x_train.shape[0] is not 0: + self._xdim = x_train.shape[1] + elif x_abs_train.shape[0] is not 0: + self._xdim = x_abs_train.shape[1] + else: + raise Exception("No Input Points") + self._nx = x_train.shape[0] + x_abs_train.shape[0] + self._n_rel = x_train.shape[0] + self._n_abs = x_abs_train.shape[0] self.x_train = x_train self.y_train = y_train + self.x_abs_train = x_abs_train + self.y_abs_train = y_abs_train self.uvi_train = uvi_train self.delta_f = delta_f - self.likelihood = likelihood() + self.x_train_all = np.concatenate((self.x_train, self.x_abs_train), 0) + + self.rel_likelihood = likelihood() + self.abs_likelihood = AbsBoundProbit() self.kern = GPy.kern.RBF(self._xdim, ARD=True) @@ -112,14 +219,16 @@ def __init__(self, x_train, uvi_train, y_train, likelihood=PrefProbit, delta_f = def calc_laplace(self, loghyp, f=None): self.kern.lengthscale = np.exp(loghyp[0:self._xdim]) self.kern.variance = (np.exp(loghyp[self._xdim]))**2 - self.likelihood.set_sigma = np.exp(loghyp[-1]) + self.rel_likelihood.set_sigma = np.exp(loghyp[-2]) # Do we need different sigmas for each likelihood? Hopefully No? + self.abs_likelihood.set_sigma = np.exp(loghyp[-2]) + #self.abs_likelihood.set_v = np.exp(loghyp[-1]) if f is None: f = np.zeros((self._nx, 1)) # With current hyperparameters: Ix = np.eye(self._nx) - K = self.kern.K(self.x_train) + K = self.kern.K(self.x_train_all) eps = 1e-6 inv_ok = False @@ -138,10 +247,36 @@ def calc_laplace(self, loghyp, f=None): f_error = self.delta_f + 1 while f_error > self.delta_f: - W, dpy_df = self.likelihood.derivatives(self.uvi_train, self.y_train, f) + # Is splitting these apart correct? Will there be off-diagonal elements of W_abs that should be + # non-zero because of the relative training points? + f_rel = f[0:self._n_rel] + f_abs = f[self._n_rel:] + + # Get relative Hessian and Gradient + if self._n_rel>0 and self._n_abs>0: + W_rel, dpy_df_rel = self.rel_likelihood.derivatives(self.uvi_train, self.y_train, f_rel) + # Get Absolute Hessian and Gradient + # Note that y_abs_train has to be [0,1], which could be an issue. + W_abs, dpy_df_abs = self.abs_likelihood.derivatives(self.y_abs_train, f_abs) + + # Combine W, gradient + dpy_df = np.concatenate((dpy_df_rel, dpy_df_abs), axis=0) + W = block_diag(W_rel, W_abs) + + elif self._n_rel>0: + W, dpy_df = self.rel_likelihood.derivatives(self.uvi_train, self.y_train, f_rel) + + elif self._n_abs>0: + W, dpy_df = self.abs_likelihood.derivatives(self.y_abs_train, f_abs) + + # print "Total" + print "Dpy, W:" + print dpy_df + print W + g = (iK + W) f_new = np.matmul(np.linalg.inv(g), np.matmul(W, f) + dpy_df) - lml = self.likelihood.log_marginal(self.uvi_train, self.y_train, f_new, iK, logdetK) + #lml = self.rel_likelihood.log_marginal(self.uvi_train, self.y_train, f_new, iK, logdetK) ## Jensen version (iK + W)^-1 = K - K((I + WK)^-1)WK (not sure how to get f'K^-1f though... # ig = K - np.matmul(np.matmul(np.matmul(K, np.linalg.inv(Ix + np.matmul(W, K))), W), K) @@ -151,10 +286,11 @@ def calc_laplace(self, loghyp, f=None): df = np.abs((f_new - f)) f_error = np.max(df) - print f_error,lml + print "F Error: " + str(f_error) #,lml + # print "F New: " + str(f_new) f = f_new - return f, lml + return f#, lml def calc_nlml(self, loghyp): f,lml = self.calc_laplace(loghyp) From fce5cf6af2e0722daadd27e0ca75a00de96b3e90 Mon Sep 17 00:00:00 2001 From: Thane Somers Date: Mon, 27 Feb 2017 18:48:15 -0800 Subject: [PATCH 02/38] Estimate Derivatives Added estimated derivatives for debugging --- GP_preference_demo.py | 2 +- GPpref.py | 18 ++++++++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/GP_preference_demo.py b/GP_preference_demo.py index f782fd6..3d710a7 100644 --- a/GP_preference_demo.py +++ b/GP_preference_demo.py @@ -5,7 +5,7 @@ import scipy.optimize as op log_hyp = np.log([0.1,1.0,0.1,10.0]) # length_scale, sigma_f, sigma_probit, v_beta -np.random.seed(2) +np.random.seed(3) n_rel_train = 0 n_abs_train = 5 diff --git a/GPpref.py b/GPpref.py index e15cc8d..d2da267 100644 --- a/GPpref.py +++ b/GPpref.py @@ -3,7 +3,7 @@ import GPy #from scipy.special import ndtr as std_norm_cdf, digamma, polygamma from scipy.special import digamma, polygamma -from scipy.stats import norm +from scipy.stats import norm, beta from scipy.linalg import block_diag #define a standard normal pdf @@ -126,6 +126,10 @@ def alpha(self, f): def beta(self, f): return self.v * (1-self.mean_link(f)) + def likelihood(self, y, f ): + ml = self.mean_link(f) + return beta.pdf(y, self.v*ml, self.v*(1-ml)) + def derivatives(self, y, f): print "Start Iter. f, y" @@ -136,6 +140,16 @@ def derivatives(self, y, f): # print "Alpha: " + str(alpha) # print "Beta: " + str(beta) + # Estimate dpy_df + delta = 0.001 + print "Estimated dpy_df" + est_dpy_df = (self.likelihood(y, f+delta) - self.likelihood(y, f-delta))/2*delta + print est_dpy_df + + print "Estimated W" + est_W = np.diagflat((self.likelihood(y, f+2*delta) - 2*self.likelihood(y,f) + self.likelihood(y, f-2*delta))/(2*delta)**2) + print est_W + # Theres a dot in Jensen that I'm hoping is just a typo. I didn't fully derive it, but it looks correct. # As the iteration goes, f blows up. This makes parts of alpha, beta go to v and 0. Digamma(0)=-inf :( # So why is it blowing up? @@ -221,7 +235,7 @@ def calc_laplace(self, loghyp, f=None): self.kern.variance = (np.exp(loghyp[self._xdim]))**2 self.rel_likelihood.set_sigma = np.exp(loghyp[-2]) # Do we need different sigmas for each likelihood? Hopefully No? self.abs_likelihood.set_sigma = np.exp(loghyp[-2]) - #self.abs_likelihood.set_v = np.exp(loghyp[-1]) + self.abs_likelihood.set_v = np.exp(loghyp[-1]) if f is None: f = np.zeros((self._nx, 1)) From c0f7a17e096e65702fcae7e1c08dda0ec7fcb36f Mon Sep 17 00:00:00 2001 From: Thane Somers Date: Mon, 27 Feb 2017 19:24:29 -0800 Subject: [PATCH 03/38] Corrected Derivatives --- GPpref.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/GPpref.py b/GPpref.py index d2da267..11ec350 100644 --- a/GPpref.py +++ b/GPpref.py @@ -128,7 +128,7 @@ def beta(self, f): def likelihood(self, y, f ): ml = self.mean_link(f) - return beta.pdf(y, self.v*ml, self.v*(1-ml)) + return np.log(beta.pdf(y, self.v*ml, self.v*(1-ml))) def derivatives(self, y, f): @@ -143,12 +143,13 @@ def derivatives(self, y, f): # Estimate dpy_df delta = 0.001 print "Estimated dpy_df" - est_dpy_df = (self.likelihood(y, f+delta) - self.likelihood(y, f-delta))/2*delta + est_dpy_df = (self.likelihood(y, f+delta) - self.likelihood(y, f-delta))/(2*delta) print est_dpy_df print "Estimated W" - est_W = np.diagflat((self.likelihood(y, f+2*delta) - 2*self.likelihood(y,f) + self.likelihood(y, f-2*delta))/(2*delta)**2) - print est_W + est_W_diag = (self.likelihood(y, f+2*delta) - 2*self.likelihood(y,f) + self.likelihood(y, f-2*delta))/(2*delta)**2 + + print est_W_diag # Theres a dot in Jensen that I'm hoping is just a typo. I didn't fully derive it, but it looks correct. # As the iteration goes, f blows up. This makes parts of alpha, beta go to v and 0. Digamma(0)=-inf :( @@ -156,8 +157,10 @@ def derivatives(self, y, f): dpy_df = self.v*std_norm_pdf(f*self._isqrt2sig) * (np.log(y)-np.log(1-y) - digamma(alpha) + digamma(beta) ) Wdiag = ( - self.v**2*std_norm_pdf(f*self._isqrt2sig) * - ( std_norm_pdf(f*self._isqrt2sig) * ( polygamma(0, alpha) + polygamma(0, alpha) ) + - f*self._i2var/self.v * (np.log(y)-np.log(1-y)-digamma(alpha) + digamma(beta)) )) + ( std_norm_pdf(f*self._isqrt2sig) * ( polygamma(1, alpha) + polygamma(1, alpha) ) + + f*self._isqrt2sig/self.v * (np.log(y)-np.log(1-y)-digamma(alpha) + digamma(beta)) )) + print "Wdiag" + print Wdiag W = np.diagflat(Wdiag) @@ -238,7 +241,8 @@ def calc_laplace(self, loghyp, f=None): self.abs_likelihood.set_v = np.exp(loghyp[-1]) if f is None: - f = np.zeros((self._nx, 1)) + f = np.ones((self._nx, 1)) + f = f*.5 # With current hyperparameters: Ix = np.eye(self._nx) From a679c9753a9714e927c40e81dc14158d0da1a9de Mon Sep 17 00:00:00 2001 From: Nicholas Lawrance Date: Tue, 28 Feb 2017 16:56:44 -0800 Subject: [PATCH 04/38] Fixed beta stuff --- GP_preference_demo.py | 11 ++++++++--- GPpref.py | 44 +++++++++++++++++++++++-------------------- 2 files changed, 32 insertions(+), 23 deletions(-) diff --git a/GP_preference_demo.py b/GP_preference_demo.py index 3d710a7..569dc1c 100644 --- a/GP_preference_demo.py +++ b/GP_preference_demo.py @@ -7,7 +7,7 @@ log_hyp = np.log([0.1,1.0,0.1,10.0]) # length_scale, sigma_f, sigma_probit, v_beta np.random.seed(3) -n_rel_train = 0 +n_rel_train = 10 n_abs_train = 5 true_sigma = 0.05 delta_f = 1e-5 @@ -89,8 +89,13 @@ def noisy_preference_rank(uv, sigma): #f,lml = prefGP.calc_laplace(log_hyp) f = prefGP.calc_laplace(log_hyp) +# New y's are expectations from Beta distribution. E(X) = alpha/(alpha+beta) +alph = prefGP.abs_likelihood.alpha(f) +bet = prefGP.abs_likelihood.beta(f) +Ey = alph/(alph+bet) + if x_train.shape[0]>0: - ha.plot(x_train, f[0:x_train.shape[0]], 'g^') + ha.plot(x_train, Ey[0:x_train.shape[0]], 'g^') if x_abs_train.shape[0]>0: - ha.plot(x_abs_train, f[x_train.shape[0]:], 'r^') + ha.plot(x_abs_train, Ey[x_train.shape[0]:], 'r^') plt.show() diff --git a/GPpref.py b/GPpref.py index 11ec350..3088aec 100644 --- a/GPpref.py +++ b/GPpref.py @@ -118,7 +118,8 @@ def set_v(self, v): self.v = v def mean_link(self, f): - return std_norm_cdf(f*self._isqrt2sig) + ml = np.clip(std_norm_cdf(f*self._isqrt2sig), 1e-12, 1.0-1e-12) + return ml def alpha(self, f): return self.v * self.mean_link(f) @@ -142,29 +143,31 @@ def derivatives(self, y, f): # Estimate dpy_df delta = 0.001 - print "Estimated dpy_df" + #print "Estimated dpy_df" est_dpy_df = (self.likelihood(y, f+delta) - self.likelihood(y, f-delta))/(2*delta) - print est_dpy_df + #print est_dpy_df + print 'log likelihood' + print np.sum(self.likelihood(y, f)) - print "Estimated W" + #print "Estimated W" est_W_diag = (self.likelihood(y, f+2*delta) - 2*self.likelihood(y,f) + self.likelihood(y, f-2*delta))/(2*delta)**2 - print est_W_diag + #print est_W_diag # Theres a dot in Jensen that I'm hoping is just a typo. I didn't fully derive it, but it looks correct. # As the iteration goes, f blows up. This makes parts of alpha, beta go to v and 0. Digamma(0)=-inf :( # So why is it blowing up? - dpy_df = self.v*std_norm_pdf(f*self._isqrt2sig) * (np.log(y)-np.log(1-y) - digamma(alpha) + digamma(beta) ) + dpy_df = self.v*self._isqrt2sig*std_norm_pdf(f*self._isqrt2sig) * (np.log(y)-np.log(1-y) - digamma(alpha) + digamma(beta) ) - Wdiag = ( - self.v**2*std_norm_pdf(f*self._isqrt2sig) * - ( std_norm_pdf(f*self._isqrt2sig) * ( polygamma(1, alpha) + polygamma(1, alpha) ) + - f*self._isqrt2sig/self.v * (np.log(y)-np.log(1-y)-digamma(alpha) + digamma(beta)) )) - print "Wdiag" - print Wdiag + Wdiag = - self.v*self._isqrt2sig*std_norm_pdf(f*self._isqrt2sig) * ( + f*self._i2var*( np.log(y)-np.log(1.0-y)-digamma(alpha) + digamma(beta) ) + + self.v*self._isqrt2sig*std_norm_pdf(f*self._isqrt2sig) * (polygamma(1, alpha) + polygamma(1, beta)) ) + # print "Wdiag" + # print Wdiag W = np.diagflat(Wdiag) - return W, dpy_df + return -W, dpy_df def log_marginal(self): pass @@ -242,7 +245,7 @@ def calc_laplace(self, loghyp, f=None): if f is None: f = np.ones((self._nx, 1)) - f = f*.5 + f = f*.0 # With current hyperparameters: Ix = np.eye(self._nx) @@ -287,13 +290,14 @@ def calc_laplace(self, loghyp, f=None): elif self._n_abs>0: W, dpy_df = self.abs_likelihood.derivatives(self.y_abs_train, f_abs) - # print "Total" - print "Dpy, W:" - print dpy_df - print W + # # print "Total" + # print "Dpy, W:" + # print dpy_df + # print W + lambda_eye = 0.0*np.eye(iK.shape[0]) - g = (iK + W) - f_new = np.matmul(np.linalg.inv(g), np.matmul(W, f) + dpy_df) + g = (iK + W - lambda_eye) + f_new = np.matmul(np.linalg.inv(g), np.matmul(W-lambda_eye, f) + dpy_df) #lml = self.rel_likelihood.log_marginal(self.uvi_train, self.y_train, f_new, iK, logdetK) ## Jensen version (iK + W)^-1 = K - K((I + WK)^-1)WK (not sure how to get f'K^-1f though... @@ -304,7 +308,7 @@ def calc_laplace(self, loghyp, f=None): df = np.abs((f_new - f)) f_error = np.max(df) - print "F Error: " + str(f_error) #,lml + # print "F Error: " + str(f_error) #,lml # print "F New: " + str(f_new) f = f_new From 862ac83633a2a0ed6d857aba967ba7ced1410d3b Mon Sep 17 00:00:00 2001 From: Nicholas Lawrance Date: Wed, 15 Mar 2017 16:35:16 -0700 Subject: [PATCH 05/38] Added sampling to recover posterior distribution for visualisation, now to add active learning --- GP_preference_demo.py | 81 +++++++++++++++++++++++++++---------- GPpref.py | 89 +++++++++++++++++++++++++++++++---------- mcmc.py | 93 +++++++++++++++++++++++++++++++++++++++++++ nice_plot_colors.py | 22 ++++++++++ plot_tools.py | 18 +++++++++ 5 files changed, 261 insertions(+), 42 deletions(-) create mode 100644 mcmc.py create mode 100644 nice_plot_colors.py create mode 100644 plot_tools.py diff --git a/GP_preference_demo.py b/GP_preference_demo.py index 569dc1c..cf3305b 100644 --- a/GP_preference_demo.py +++ b/GP_preference_demo.py @@ -3,18 +3,27 @@ import matplotlib.pyplot as plt import GPpref import scipy.optimize as op +import plot_tools as ptt +import mcmc +from scipy.stats import beta +plt.rc('font',**{'family':'serif','sans-serif':['Computer Modern Roman']}) +plt.rc('text', usetex=True) -log_hyp = np.log([0.1,1.0,0.1,10.0]) # length_scale, sigma_f, sigma_probit, v_beta +log_hyp = np.log([0.1,0.5,0.1,10.0]) # length_scale, sigma_f, sigma_probit, v_beta np.random.seed(3) n_rel_train = 10 -n_abs_train = 5 +n_abs_train = 10 true_sigma = 0.05 delta_f = 1e-5 +n_xplot = 101 +n_mcsamples = 1000 +n_ysamples = 101 + # Define polynomial function to be modelled def true_function(x): - y = (np.sin(x*2*np.pi + np.pi/4) + 1)/2 + y = (np.sin(x*2*np.pi + np.pi/4) + 1.25)/2.5 #y = np.sin(x*2.0*np.pi + np.pi/4.0) return y @@ -31,12 +40,10 @@ def noisy_preference_rank(uv, sigma): return y, fuv # Main program -# Plot true function -x_plot = np.linspace(0.0,1.0,101,dtype='float') +# True function +x_plot = np.linspace(0.0,1.0,n_xplot,dtype='float') y_plot = true_function(x_plot) -hf,ha = plt.subplots(1,1) -ha.plot(x_plot,y_plot,'r-') - +x_test = np.atleast_2d(x_plot).T # Training data - this is a bit weird, but we sample x values, then the uv pairs # are actually indexes into x, because it is easier computationally. You can @@ -48,9 +55,6 @@ def noisy_preference_rank(uv, sigma): # Get noisy observations f(uv) and corresponding ranks y_train y_train, fuv_train = noisy_preference_rank(uv_train, true_sigma) - for uv,fuv,y in zip(uv_train, fuv_train, y_train): - ha.plot(uv, fuv, 'b-') - ha.plot(uv[(y+1)/2],fuv[(y+1)/2],'k+') else: x_train = np.zeros((0,1)) @@ -63,11 +67,7 @@ def noisy_preference_rank(uv, sigma): #y_abs_train = obs_function(x_abs_train, true_sigma) y_abs_train = np.clip(obs_function(x_abs_train, true_sigma), 0.01, .99) -ha.plot(x_abs_train, y_abs_train, 'r+') -ha.set_title('Training data') -ha.set_ylabel('f(x)') -ha.set_xlabel('x') # GP hyperparameters # note: using log scaling to aid learning hyperparameters with varied magnitudes @@ -89,13 +89,54 @@ def noisy_preference_rank(uv, sigma): #f,lml = prefGP.calc_laplace(log_hyp) f = prefGP.calc_laplace(log_hyp) +# Latent predictions +fhat, vhat = prefGP.predict_latent(x_test) +vhat = np.atleast_2d(vhat.diagonal()).T + +# Expected values +E_y = prefGP.expected_y(x_test, fhat, vhat) + +# Sampling from posterior to show likelihoods +p_y = np.zeros((n_ysamples, n_xplot)) +y_samples = np.linspace(0.0, 1.0, n_ysamples) +iny = 1.0/n_ysamples +E_y2 = np.zeros(n_xplot) + +normal_samples = np.random.normal(size=n_mcsamples) +for i,(fstar,vstar) in enumerate(zip(fhat, vhat)): + f_samples = normal_samples*vstar+fstar + aa, bb = prefGP.abs_likelihood.get_alpha_beta(f_samples) + p_y[:,i] = [iny*np.sum(beta.pdf(yj, aa, bb)) for yj in y_samples] + E_y2[i] = np.sum(np.dot(y_samples, p_y[:,i]))/np.sum(p_y[:,i]) + # New y's are expectations from Beta distribution. E(X) = alpha/(alpha+beta) -alph = prefGP.abs_likelihood.alpha(f) -bet = prefGP.abs_likelihood.beta(f) -Ey = alph/(alph+bet) +#alph = prefGP.abs_likelihood.alpha(f) +#bet = prefGP.abs_likelihood.beta(f) +#Ey = alph/(alph+bet) + +hf, (ha, hb) = plt.subplots(1,2) +hf, hpf = ptt.plot_with_bounds(ha, x_plot, y_plot, true_sigma, c=ptt.lines[0]) if x_train.shape[0]>0: - ha.plot(x_train, Ey[0:x_train.shape[0]], 'g^') + for uv,fuv,y in zip(uv_train, fuv_train, y_train): + ha.plot(uv, fuv, 'b-', color=ptt.lighten(ptt.lines[0])) + ha.plot(uv[(y+1)/2],fuv[(y+1)/2],'+', color=ptt.darken(ptt.lines[0], 1.5)) + if x_abs_train.shape[0]>0: - ha.plot(x_abs_train, Ey[x_train.shape[0]:], 'r^') + ha.plot(x_abs_train, y_abs_train, '+', color=ptt.lighten(ptt.lines[2])) + +hfhat, hpfhat = ptt.plot_with_bounds(hb, x_test, fhat, np.sqrt(vhat), c=ptt.lines[1]) +hEy, = ha.plot(x_plot, E_y, color=ptt.lines[3]) + +ha.imshow(p_y, origin='lower', extent=[x_plot[0], x_plot[-1], 0.0, 1.0]) +# ha.plot(x_plot, E_y2, color=ptt.lines[4]) +hmap, = ha.plot(x_plot, y_samples[np.argmax(p_y, axis=0)], color='w') +ha.set_title('Training data') +ha.set_ylabel('$y$') +ha.set_xlabel('$x$') +hb.set_xlabel('$x$') +hb.set_ylabel('$f(x)$') +ha.legend([hf, hEy, hmap], [r'$f(x)$', r'$\mathbb{E}_{p(y|\mathcal{Y})}\left[y\right]$', r'$y_{MAP} | \mathcal{Y}$']) +hb.legend([hfhat], [r'Latent function $\hat{f}(x)$']) + plt.show() diff --git a/GPpref.py b/GPpref.py index 3088aec..a6caf7e 100644 --- a/GPpref.py +++ b/GPpref.py @@ -127,9 +127,18 @@ def alpha(self, f): def beta(self, f): return self.v * (1-self.mean_link(f)) - def likelihood(self, y, f ): + def get_alpha_beta(self, f): ml = self.mean_link(f) - return np.log(beta.pdf(y, self.v*ml, self.v*(1-ml))) + aa = self.v * ml + bb = self.v * (1-ml) + return aa, bb + + def likelihood(self, y, f): + aa, bb = self.get_alpha_beta(f) + return beta.pdf(y, aa, bb) + + def log_likelihood(self, y, f): + return np.log(self.likelihood(y,f)) def derivatives(self, y, f): @@ -144,13 +153,13 @@ def derivatives(self, y, f): # Estimate dpy_df delta = 0.001 #print "Estimated dpy_df" - est_dpy_df = (self.likelihood(y, f+delta) - self.likelihood(y, f-delta))/(2*delta) + #est_dpy_df = (self.log_likelihood(y, f+delta) - self.log_likelihood(y, f-delta))/(2*delta) #print est_dpy_df print 'log likelihood' - print np.sum(self.likelihood(y, f)) + print np.sum(self.log_likelihood(y, f)) #print "Estimated W" - est_W_diag = (self.likelihood(y, f+2*delta) - 2*self.likelihood(y,f) + self.likelihood(y, f-2*delta))/(2*delta)**2 + #est_W_diag = (self.log_likelihood(y, f+2*delta) - 2*self.log_likelihood(y,f) + self.log_likelihood(y, f-2*delta))/(2*delta)**2 #print est_W_diag @@ -172,6 +181,10 @@ def derivatives(self, y, f): def log_marginal(self): pass + def expectation(self, fhat, var_star): + #mu_t = self.mean_link(fhat) + E_x = np.clip(std_norm_cdf(fhat/(np.sqrt(2*self.sigma**2 + var_star))), 1e-12, 1.0-1e-12) + return E_x class AbsProbit(object): # The Probit Likelihood given in Rasmussen, p43. Note f(x) is scaled by sqrt(2)*sigma, as in Jensen. @@ -235,6 +248,8 @@ def __init__(self, x_train, uvi_train, x_abs_train, y_train, y_abs_train, likeli self.kern = GPy.kern.RBF(self._xdim, ARD=True) + self.Ix = np.eye(self._nx) + def calc_laplace(self, loghyp, f=None): self.kern.lengthscale = np.exp(loghyp[0:self._xdim]) @@ -248,21 +263,9 @@ def calc_laplace(self, loghyp, f=None): f = f*.0 # With current hyperparameters: - Ix = np.eye(self._nx) - K = self.kern.K(self.x_train_all) - eps = 1e-6 - inv_ok = False + self.Kxx = self.kern.K(self.x_train_all) - while not inv_ok: - try: - L = np.linalg.cholesky(K + eps*Ix) - iK = np.linalg.solve(L.T, np.linalg.solve(L, Ix)) - # detK = (np.product(L.diagonal()))**2 - logdetK = np.sum(np.log(L.diagonal())) - inv_ok = True - except np.linalg.linalg.LinAlgError: - eps = eps*10 - print "Inversion issue, adding noise: {0}".format(eps) + self.iK, logdetK = self._safe_invert_noise(self.Kxx) # First, solve for \hat{f} and W (mode finding Laplace approximation, Newton-Raphson) f_error = self.delta_f + 1 @@ -294,9 +297,9 @@ def calc_laplace(self, loghyp, f=None): # print "Dpy, W:" # print dpy_df # print W - lambda_eye = 0.0*np.eye(iK.shape[0]) + lambda_eye = 0.0*np.eye(self.iK.shape[0]) - g = (iK + W - lambda_eye) + g = (self.iK + W - lambda_eye) f_new = np.matmul(np.linalg.inv(g), np.matmul(W-lambda_eye, f) + dpy_df) #lml = self.rel_likelihood.log_marginal(self.uvi_train, self.y_train, f_new, iK, logdetK) @@ -312,8 +315,50 @@ def calc_laplace(self, loghyp, f=None): # print "F New: " + str(f_new) f = f_new + self.W = W + self.f = f + self.iKf = np.matmul(self.iK, self.f) + return f#, lml + def _safe_invert_noise(self, mat): + eps = 1e-6 + inv_ok = False + + while not inv_ok: + try: + L = np.linalg.cholesky(mat + eps*self.Ix) + imat = np.linalg.solve(L.T, np.linalg.solve(L, self.Ix)) + # detK = (np.product(L.diagonal()))**2 + logdet = np.sum(np.log(L.diagonal())) + inv_ok = True + except np.linalg.linalg.LinAlgError: + eps = eps*10 + print "Inversion issue, adding noise: {0}".format(eps) + return imat, logdet + def calc_nlml(self, loghyp): f,lml = self.calc_laplace(loghyp) - return -lml \ No newline at end of file + return -lml + + def predict_latent(self, x): + assert hasattr(self, 'iKf') + kt = self.kern.K(self.x_train_all, x) + mean_latent = np.matmul(kt.T, self.iKf) + Ktt = self.kern.K(x) + iKW = np.linalg.inv(np.matmul(self.W, self.Kxx) + self.Ix) + var_latent = Ktt - np.matmul(kt.T, np.matmul(iKW, np.matmul(self.W, kt))) + return mean_latent, var_latent + + def expected_y(self, x, fhat=None, varhat=None): + if fhat is None or varhat is None: + fhat, varhat = self.predict_latent(x) + #Ktt = self.kern.K(x) + #var_star = 2*self.abs_likelihood.sigma**2 + np.atleast_2d(Ktt.diagonal()).T + E_y = self.abs_likelihood.expectation(fhat, np.atleast_2d(varhat.diagonal()).T) + return E_y + + + + + diff --git a/mcmc.py b/mcmc.py new file mode 100644 index 0000000..923a53b --- /dev/null +++ b/mcmc.py @@ -0,0 +1,93 @@ +import numpy as np + + +class Enum(set): + def __getattr__(self, name): + if name in self: + return name + raise AttributeError + + +Samplers = Enum(['Metropolis', 'MetropolisHastings', 'Gibbs']) + + +class MCMCSampler(object): + def __init__(self, func, limits, q=None, seed=None, func_kwargs={}): + self.f = func + # self.sampler = sampler ## , sampler=Samplers.'MetropolisHastings' + self.limits = limits + self.lim_range = self.limits[1, :] - self.limits[0, :] + self.n_dim = self.limits.shape[1] + np.random.seed(seed) + if q == None: + q_std = self.lim_range / 10.0 + q = (lambda x: x + np.random.normal(size=self.n_dim) * q_std) + self.q = q + self.func_kwargs = func_kwargs + + @property + def sampler(self): + return self._sampler + + @sampler.setter + def sampler(self, s): + if s not in Samplers: raise Exception("Specified sampler must be in Sampler set") + self._sampler = s + + @property + def limits(self): + return self._limits + + @limits.setter + def limits(self, l): + l = np.array(l) + if l.shape[0] != 2: raise Exception("Limits must be a 2*n_dim array") + self._limits = l + + def generate_sample(self, X_current, f_current): + # Generate proposed sample + X_proposed = self.q(X_current) + while True: + inlimits = np.logical_and(X_proposed >= self.limits[0, :], X_proposed <= self.limits[1, :]) + if not inlimits.all(): + X_proposed = self.q(X_current) + else: + break + + # Acceptance ratio + f_proposed = self.f(X_proposed, **self.func_kwargs) + alpha = f_proposed / f_current + + # Accept? + pp = np.random.rand(1) + if pp < alpha: # Accept + return X_proposed, f_proposed + return X_current, f_current + + def sample_chain(self, n_samples, burn_in=None, X_start=None): + if burn_in is None: + burn_in = np.floor_divide(n_samples, 10) + if X_start is None: + X_start = self.limits[0, :] + np.random.rand(self.n_dim) * self.lim_range + X_current = X_start + f_current = self.f(X_current, **self.func_kwargs) + + # Run for burn-in and throw away samples + for t in range(burn_in): + X_current, f_current = self.generate_sample(X_current, f_current) + if t % 100 == 0: + print "Burn-in: {0}/{1}".format(t, burn_in) + + X = np.zeros((n_samples, self.n_dim)) + f_X = np.zeros(n_samples) + X[0, :] = X_current + f_X[0] = f_current + + for t in range(1, n_samples): + X_current, f_current = self.generate_sample(X_current, f_current) + X[t, :] = X_current + f_X[t] = f_current + if t % 100 == 0: + print "Chain sampling: {0}/{1}".format(t, n_samples) + + return X, f_X \ No newline at end of file diff --git a/nice_plot_colors.py b/nice_plot_colors.py new file mode 100644 index 0000000..30e946a --- /dev/null +++ b/nice_plot_colors.py @@ -0,0 +1,22 @@ +import numpy as np + +lines = np.array([[57,106,177], [218,124,48], [62,150,81], [204,37,41], [83,81,84], [107,76,154], [146,36,40], [148,139,61]], dtype='float')/255 +bars = np.array([[114,147,203], [225,151,76], [132,186,91], [211,94,96], [128,133,133], [144,103,167], [171,104,87], [204,194,16]], dtype='float')/255 + +# These look better in print +vlines = ['cornflowerblue', 'green', 'firebrick', 'orange', 'black', 'indigo'] +vbars = ['steelblue', 'darkgreen', 'darkred', 'darkorange', 'grey', 'mediumvioletred'] + +lines = np.hstack((lines, np.ones((lines.shape[0],1)))) +bars = np.hstack((bars, np.ones((bars.shape[0],1)))) + +def darken(c,power=2): + co = np.array(c).copy() + co = np.clip(co**power, 0, 1.0) + co[-1] = c[-1] + return co + +def lighten(c, power=2): + co = 1.0-np.array(c).copy() + co = darken(co, power) + return 1.0-co \ No newline at end of file diff --git a/plot_tools.py b/plot_tools.py new file mode 100644 index 0000000..3b29036 --- /dev/null +++ b/plot_tools.py @@ -0,0 +1,18 @@ +import numpy as np +import matplotlib.pyplot as plt +from matplotlib.patches import Polygon +from nice_plot_colors import * + +def make_poly_array(x,y,sigma): + nx = len(x) + xy = np.zeros((2*nx, 2)) + xy[:,0] = np.append(x, x[::-1]) + xy[:,1] = np.append(y-sigma, y[::-1]+sigma) + return xy + + +def plot_with_bounds(ax, x, y, s, c=lines[0]): + h_patch = Polygon(make_poly_array(x, y, s), ec=c, fc=lighten(c, 3), alpha=0.5) + h_fx, = ax.plot(x, y, lw=1.5, c=c) + ax.add_patch(h_patch) + return h_fx, h_patch \ No newline at end of file From 9689379db7f4f43de3b9b979e938bdbe06d3bae5 Mon Sep 17 00:00:00 2001 From: Nicholas Lawrance Date: Thu, 16 Mar 2017 18:05:22 -0700 Subject: [PATCH 06/38] WIP, active learning doesn't do anything, tried to add beta in input to pref demo but not really working. --- GP_preference_demo.py | 43 +++++++++--- GPpref.py | 6 +- pref_active_learning.py | 145 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 180 insertions(+), 14 deletions(-) create mode 100644 pref_active_learning.py diff --git a/GP_preference_demo.py b/GP_preference_demo.py index cf3305b..83007c1 100644 --- a/GP_preference_demo.py +++ b/GP_preference_demo.py @@ -27,14 +27,35 @@ def true_function(x): #y = np.sin(x*2.0*np.pi + np.pi/4.0) return y -# Define noisy observation function -def obs_function(x, sigma): +class ObservationFunction(object): + def __init__(self, true_fun): + self.f = true_fun + + def generate_observations(self, x): + fx = self.f(x) + + + + +# Gaussian noise observation function +def normal_obs_function(x): fx = true_function(x) - noise = np.random.normal(scale=sigma, size=x.shape) + noise = np.random.normal(scale=true_sigma, size=x.shape) return fx + noise - -def noisy_preference_rank(uv, sigma): - fuv = obs_function(uv,sigma) + +beta_obs = GPpref.AbsBoundProbit(sigma=true_sigma, v=10.0) + +def beta_obs_function(x): + fx = true_function(x) + a, b = beta_obs.get_alpha_beta(fx) + z = [beta.rvs(aa, bb) for aa,bb in zip(a,b)] + return z + +# Set target observation function +obs_function = normal_obs_function + +def noisy_preference_rank(uv): + fuv = obs_function(uv) y = -1*np.ones((fuv.shape[0],1),dtype='int') y[fuv[:,1] > fuv[:,0]] = 1 return y, fuv @@ -54,7 +75,7 @@ def noisy_preference_rank(uv, sigma): uv_train = x_train[uvi_train][:,:,0] # Get noisy observations f(uv) and corresponding ranks y_train - y_train, fuv_train = noisy_preference_rank(uv_train, true_sigma) + y_train, fuv_train = noisy_preference_rank(uv_train) else: x_train = np.zeros((0,1)) @@ -65,7 +86,7 @@ def noisy_preference_rank(uv, sigma): x_abs_train = np.random.random((n_abs_train,1)) #y_abs_train = obs_function(x_abs_train, true_sigma) -y_abs_train = np.clip(obs_function(x_abs_train, true_sigma), 0.01, .99) +y_abs_train = np.clip(obs_function(x_abs_train), 0.01, .99) @@ -79,7 +100,7 @@ def noisy_preference_rank(uv, sigma): print y_abs_train # print uvi_train -prefGP = GPpref.PreferenceGaussianProcess(x_train, uvi_train, x_abs_train, y_train, y_abs_train, delta_f=delta_f) +prefGP = GPpref.PreferenceGaussianProcess(x_train, uvi_train, x_abs_train, y_train, y_abs_train, delta_f=delta_f,abs_likelihood=GPpref.AbsBoundProbit(sigma=0.5, v=10.0)) # Pseudocode: # FOr a set of hyperparameters, return log likelihood that can be used by an optimiser @@ -98,7 +119,7 @@ def noisy_preference_rank(uv, sigma): # Sampling from posterior to show likelihoods p_y = np.zeros((n_ysamples, n_xplot)) -y_samples = np.linspace(0.0, 1.0, n_ysamples) +y_samples = np.linspace(0.01, 0.99, n_ysamples) iny = 1.0/n_ysamples E_y2 = np.zeros(n_xplot) @@ -128,7 +149,7 @@ def noisy_preference_rank(uv, sigma): hfhat, hpfhat = ptt.plot_with_bounds(hb, x_test, fhat, np.sqrt(vhat), c=ptt.lines[1]) hEy, = ha.plot(x_plot, E_y, color=ptt.lines[3]) -ha.imshow(p_y, origin='lower', extent=[x_plot[0], x_plot[-1], 0.0, 1.0]) +ha.imshow(p_y, origin='lower', extent=[x_plot[0], x_plot[-1], 0.01, 0.99]) # ha.plot(x_plot, E_y2, color=ptt.lines[4]) hmap, = ha.plot(x_plot, y_samples[np.argmax(p_y, axis=0)], color='w') ha.set_title('Training data') diff --git a/GPpref.py b/GPpref.py index a6caf7e..85cc063 100644 --- a/GPpref.py +++ b/GPpref.py @@ -222,7 +222,7 @@ def derivatives(self, y, f): class PreferenceGaussianProcess(object): - def __init__(self, x_train, uvi_train, x_abs_train, y_train, y_abs_train, likelihood=PrefProbit, delta_f = 1e-6): + def __init__(self, x_train, uvi_train, x_abs_train, y_train, y_abs_train, likelihood=PrefProbit(), delta_f = 1e-6, abs_likelihood=AbsBoundProbit()): # log_hyp are log of hyperparameters, note that it is [length_0, ..., length_d, sigma_f, sigma_probit, v_beta] # Training points are split into relative and absolute for calculating f, but combined for predictions. if x_train.shape[0] is not 0: @@ -243,8 +243,8 @@ def __init__(self, x_train, uvi_train, x_abs_train, y_train, y_abs_train, likeli self.x_train_all = np.concatenate((self.x_train, self.x_abs_train), 0) - self.rel_likelihood = likelihood() - self.abs_likelihood = AbsBoundProbit() + self.rel_likelihood = likelihood + self.abs_likelihood = abs_likelihood self.kern = GPy.kern.RBF(self._xdim, ARD=True) diff --git a/pref_active_learning.py b/pref_active_learning.py new file mode 100644 index 0000000..c9cf94c --- /dev/null +++ b/pref_active_learning.py @@ -0,0 +1,145 @@ +# Simple 1D GP classification example +import numpy as np +import matplotlib.pyplot as plt +import GPpref +import scipy.optimize as op +import plot_tools as ptt +import mcmc +from scipy.stats import beta + +plt.rc('font', **{'family': 'serif', 'sans-serif': ['Computer Modern Roman']}) +plt.rc('text', usetex=True) + +log_hyp = np.log([0.1, 0.5, 0.1, 10.0]) # length_scale, sigma_f, sigma_probit, v_beta +np.random.seed(3) + +n_rel_train = 0 +n_abs_train = 0 +true_sigma = 0.05 +delta_f = 1e-5 + +n_xplot = 101 +n_mcsamples = 1000 +n_ysamples = 101 + + +# Define polynomial function to be modelled +def true_function(x): + y = (np.sin(x * 2 * np.pi + np.pi / 4) + 1.25) / 2.5 + # y = np.sin(x*2.0*np.pi + np.pi/4.0) + return y + + +# Define noisy observation function +def obs_function(x, sigma): + fx = true_function(x) + noise = np.random.normal(scale=sigma, size=x.shape) + return fx + noise + + +def noisy_preference_rank(uv, sigma): + fuv = obs_function(uv, sigma) + y = -1 * np.ones((fuv.shape[0], 1), dtype='int') + y[fuv[:, 1] > fuv[:, 0]] = 1 + return y, fuv + + +# Main program +# True function +x_plot = np.linspace(0.0, 1.0, n_xplot, dtype='float') +y_plot = true_function(x_plot) +x_test = np.atleast_2d(x_plot).T + +# Training data - this is a bit weird, but we sample x values, then the uv pairs +# are actually indexes into x, because it is easier computationally. You can +# recover the actual u,v values using x[ui],x[vi] +if n_rel_train > 0: + x_train = np.random.random((2 * n_rel_train, 1)) + uvi_train = np.random.choice(range(2 * n_rel_train), (n_rel_train, 2), replace=False) + uv_train = x_train[uvi_train][:, :, 0] + + # Get noisy observations f(uv) and corresponding ranks y_train + y_train, fuv_train = noisy_preference_rank(uv_train, true_sigma) + +else: + x_train = np.zeros((0, 1)) + uvi_train = np.zeros((0, 2)) + uv_train = np.zeros((0, 2)) + y_train = np.zeros((0, 1)) + fuv_train = np.zeros((0, 2)) + +x_abs_train = np.random.random((n_abs_train, 1)) +# y_abs_train = obs_function(x_abs_train, true_sigma) +y_abs_train = np.clip(obs_function(x_abs_train, true_sigma), 0.01, .99) + +# GP hyperparameters +# note: using log scaling to aid learning hyperparameters with varied magnitudes + +print "Data" +# print x_train +print x_abs_train +# print y_train +print y_abs_train +# print uvi_train + +prefGP = GPpref.PreferenceGaussianProcess(x_train, uvi_train, x_abs_train, y_train, y_abs_train, delta_f=delta_f) + +# Pseudocode: +# FOr a set of hyperparameters, return log likelihood that can be used by an optimiser +theta0 = log_hyp + +# log_hyp = op.fmin(prefGP.calc_nlml,theta0) +# f,lml = prefGP.calc_laplace(log_hyp) +f = prefGP.calc_laplace(log_hyp) + +# Latent predictions +fhat, vhat = prefGP.predict_latent(x_test) +vhat = np.atleast_2d(vhat.diagonal()).T + +# Expected values +E_y = prefGP.expected_y(x_test, fhat, vhat) + +# Sampling from posterior to show likelihoods +p_y = np.zeros((n_ysamples, n_xplot)) +y_samples = np.linspace(0.0, 1.0, n_ysamples) +iny = 1.0 / n_ysamples +E_y2 = np.zeros(n_xplot) + +normal_samples = np.random.normal(size=n_mcsamples) +for i, (fstar, vstar) in enumerate(zip(fhat, vhat)): + f_samples = normal_samples * vstar + fstar + aa, bb = prefGP.abs_likelihood.get_alpha_beta(f_samples) + p_y[:, i] = [iny * np.sum(beta.pdf(yj, aa, bb)) for yj in y_samples] + E_y2[i] = np.sum(np.dot(y_samples, p_y[:, i])) / np.sum(p_y[:, i]) + +# New y's are expectations from Beta distribution. E(X) = alpha/(alpha+beta) +# alph = prefGP.abs_likelihood.alpha(f) +# bet = prefGP.abs_likelihood.beta(f) +# Ey = alph/(alph+bet) + +hf, (ha, hb) = plt.subplots(1, 2) +hf, hpf = ptt.plot_with_bounds(ha, x_plot, y_plot, true_sigma, c=ptt.lines[0]) + +if x_train.shape[0] > 0: + for uv, fuv, y in zip(uv_train, fuv_train, y_train): + ha.plot(uv, fuv, 'b-', color=ptt.lighten(ptt.lines[0])) + ha.plot(uv[(y + 1) / 2], fuv[(y + 1) / 2], '+', color=ptt.darken(ptt.lines[0], 1.5)) + +if x_abs_train.shape[0] > 0: + ha.plot(x_abs_train, y_abs_train, '+', color=ptt.lighten(ptt.lines[2])) + +hfhat, hpfhat = ptt.plot_with_bounds(hb, x_test, fhat, np.sqrt(vhat), c=ptt.lines[1]) +hEy, = ha.plot(x_plot, E_y, color=ptt.lines[3]) + +ha.imshow(p_y, origin='lower', extent=[x_plot[0], x_plot[-1], 0.0, 1.0]) +# ha.plot(x_plot, E_y2, color=ptt.lines[4]) +hmap, = ha.plot(x_plot, y_samples[np.argmax(p_y, axis=0)], color='w') +ha.set_title('Training data') +ha.set_ylabel('$y$') +ha.set_xlabel('$x$') +hb.set_xlabel('$x$') +hb.set_ylabel('$f(x)$') +ha.legend([hf, hEy, hmap], [r'$f(x)$', r'$\mathbb{E}_{p(y|\mathcal{Y})}\left[y\right]$', r'$y_{MAP} | \mathcal{Y}$']) +hb.legend([hfhat], [r'Latent function $\hat{f}(x)$']) + +plt.show() From 2280a14ba82bcff67a7db7ea44f75bcb2d63dc03 Mon Sep 17 00:00:00 2001 From: Nicholas Lawrance Date: Fri, 17 Mar 2017 16:49:00 -0700 Subject: [PATCH 07/38] Messing with abs probit sigma nd v for no particlar reason --- GP_preference_demo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GP_preference_demo.py b/GP_preference_demo.py index 83007c1..fa26623 100644 --- a/GP_preference_demo.py +++ b/GP_preference_demo.py @@ -100,7 +100,7 @@ def noisy_preference_rank(uv): print y_abs_train # print uvi_train -prefGP = GPpref.PreferenceGaussianProcess(x_train, uvi_train, x_abs_train, y_train, y_abs_train, delta_f=delta_f,abs_likelihood=GPpref.AbsBoundProbit(sigma=0.5, v=10.0)) +prefGP = GPpref.PreferenceGaussianProcess(x_train, uvi_train, x_abs_train, y_train, y_abs_train, delta_f=delta_f,abs_likelihood=GPpref.AbsBoundProbit(sigma=0.5, v=20.0)) # Pseudocode: # FOr a set of hyperparameters, return log likelihood that can be used by an optimiser From a805799d39a67dae6e10a9c038cd07f3f0a63ec2 Mon Sep 17 00:00:00 2001 From: Nicholas Lawrance Date: Tue, 28 Mar 2017 12:14:45 -0700 Subject: [PATCH 08/38] Added active learning and fixed the nlml calculation (I should have made more commits...) --- GP_preference_demo.py | 7 +- GPpref.py | 84 +++++++++++++++------ active_learners.py | 113 ++++++++++++++++++++++++++++ plot_tools.py | 10 ++- pref_active_learning.py | 162 +++++++++++++++++++++------------------- 5 files changed, 270 insertions(+), 106 deletions(-) create mode 100644 active_learners.py diff --git a/GP_preference_demo.py b/GP_preference_demo.py index fa26623..02582cf 100644 --- a/GP_preference_demo.py +++ b/GP_preference_demo.py @@ -106,7 +106,7 @@ def noisy_preference_rank(uv): # FOr a set of hyperparameters, return log likelihood that can be used by an optimiser theta0 = log_hyp -# log_hyp = op.fmin(prefGP.calc_nlml,theta0) +log_hyp = op.fmin(prefGP.calc_nlml,theta0) #f,lml = prefGP.calc_laplace(log_hyp) f = prefGP.calc_laplace(log_hyp) @@ -127,8 +127,9 @@ def noisy_preference_rank(uv): for i,(fstar,vstar) in enumerate(zip(fhat, vhat)): f_samples = normal_samples*vstar+fstar aa, bb = prefGP.abs_likelihood.get_alpha_beta(f_samples) - p_y[:,i] = [iny*np.sum(beta.pdf(yj, aa, bb)) for yj in y_samples] - E_y2[i] = np.sum(np.dot(y_samples, p_y[:,i]))/np.sum(p_y[:,i]) + p_y[:, i] = [iny*np.sum(beta.pdf(yj, aa, bb)) for yj in y_samples] + p_y[:, i] /= np.sum(p_y[:, i]) + E_y2[i] = np.sum(np.dot(y_samples, p_y[:, i])) # New y's are expectations from Beta distribution. E(X) = alpha/(alpha+beta) #alph = prefGP.abs_likelihood.alpha(f) diff --git a/GPpref.py b/GPpref.py index 85cc063..926e00c 100644 --- a/GPpref.py +++ b/GPpref.py @@ -61,7 +61,7 @@ def set_sigma(self, sigma): self._isqrt2sig = 1.0 / (self.sigma * np.sqrt(2.0)) self._i2var = self._isqrt2sig**2 - def z_k(self, uvi, f, y): + def z_k(self, uvi, y, f): zc = self._isqrt2sig * (f[uvi[:, 1]] - f[uvi[:, 0]]) return y * zc @@ -75,7 +75,7 @@ def I_k(self, x, uv): # Jensen and Nielsen def derivatives(self, uvi, y, f): nx = len(f) - z = self.z_k(uvi, f, y) + z = self.z_k(uvi, y=y, f=f) phi_z = std_norm_cdf(z) N_z = std_norm_pdf(z) @@ -96,11 +96,18 @@ def derivatives(self, uvi, y, f): return W, dpy_df - def log_marginal(self, uvi, y, f, iK, logdetK): - z = self.z_k(uvi, f, y) + def likelihood(self, uvi, y, f): + z = self.z_k(uvi, y=y, f=f) phi_z = std_norm_cdf(z) - psi = np.sum(np.log(phi_z)) - 0.5 * np.matmul(np.matmul(f.T, iK), f) - 0.5*logdetK - iK.shape[0]/2.0*self.log2pi - return psi.flat[0] + return phi_z + + def log_likelihood(self, uvi, y, f): + return np.log(self.likelihood(uvi, y, f)) + + def prediction(self, fhat, varhat, uvi): + var_star = 2*self.sigma**2 + np.atleast_2d([varhat[u, u] + varhat[v, v] - varhat[u, v] - varhat[v, v] for u,v in uvi]).T + p_y = std_norm_cdf( (fhat[uvi[:,0]] - fhat[uvi[:,1]])/np.sqrt(var_star) ) + return p_y class AbsBoundProbit(object): def __init__(self, sigma=1.0, v=10.0): @@ -142,9 +149,9 @@ def log_likelihood(self, y, f): def derivatives(self, y, f): - print "Start Iter. f, y" - print f - print y + #print "Start Iter. f, y" + #print f + #print y alpha = self.alpha(f) beta = self.beta(f) #let's make a distribution called beta that also has beta as a parameter! # print "Alpha: " + str(alpha) @@ -155,8 +162,8 @@ def derivatives(self, y, f): #print "Estimated dpy_df" #est_dpy_df = (self.log_likelihood(y, f+delta) - self.log_likelihood(y, f-delta))/(2*delta) #print est_dpy_df - print 'log likelihood' - print np.sum(self.log_likelihood(y, f)) + #print 'log likelihood' + #print np.sum(self.log_likelihood(y, f)) #print "Estimated W" #est_W_diag = (self.log_likelihood(y, f+2*delta) - 2*self.log_likelihood(y,f) + self.log_likelihood(y, f-2*delta))/(2*delta)**2 @@ -181,7 +188,7 @@ def derivatives(self, y, f): def log_marginal(self): pass - def expectation(self, fhat, var_star): + def prediction(self, fhat, var_star): #mu_t = self.mean_link(fhat) E_x = np.clip(std_norm_cdf(fhat/(np.sqrt(2*self.sigma**2 + var_star))), 1e-12, 1.0-1e-12) return E_x @@ -225,13 +232,23 @@ class PreferenceGaussianProcess(object): def __init__(self, x_train, uvi_train, x_abs_train, y_train, y_abs_train, likelihood=PrefProbit(), delta_f = 1e-6, abs_likelihood=AbsBoundProbit()): # log_hyp are log of hyperparameters, note that it is [length_0, ..., length_d, sigma_f, sigma_probit, v_beta] # Training points are split into relative and absolute for calculating f, but combined for predictions. + self.set_observations(x_train, uvi_train, x_abs_train, y_train, y_abs_train) + + self.delta_f = delta_f + self.rel_likelihood = likelihood + self.abs_likelihood = abs_likelihood + + self.kern = GPy.kern.RBF(self._xdim, ARD=True) + + self.Ix = np.eye(self._nx) + + def set_observations(self, x_train, uvi_train, x_abs_train, y_train, y_abs_train): if x_train.shape[0] is not 0: self._xdim = x_train.shape[1] elif x_abs_train.shape[0] is not 0: self._xdim = x_abs_train.shape[1] - else: + else: raise Exception("No Input Points") - self._nx = x_train.shape[0] + x_abs_train.shape[0] self._n_rel = x_train.shape[0] self._n_abs = x_abs_train.shape[0] self.x_train = x_train @@ -239,17 +256,24 @@ def __init__(self, x_train, uvi_train, x_abs_train, y_train, y_abs_train, likeli self.x_abs_train = x_abs_train self.y_abs_train = y_abs_train self.uvi_train = uvi_train - self.delta_f = delta_f - self.x_train_all = np.concatenate((self.x_train, self.x_abs_train), 0) - self.rel_likelihood = likelihood - self.abs_likelihood = abs_likelihood - - self.kern = GPy.kern.RBF(self._xdim, ARD=True) + self.x_train_all = np.concatenate((self.x_train, self.x_abs_train), 0) + self._nx = self.x_train_all.shape[0] self.Ix = np.eye(self._nx) + def add_observations(self, x, y, uvi=None): + if uvi is None: + x_abs_train = np.concatenate((self.x_abs_train, x), 0) + y_abs_train = np.concatenate((self.y_abs_train, y), 0) + self.set_observations(self.x_train, self.uvi_train, x_abs_train, self.y_train, y_abs_train) + else: + x_train = np.concatenate((self.x_train, x), 0) + y_train = np.concatenate((self.y_train, y), 0) + uvi_train = np.concatenate((self.uvi_train, uvi+self.x_train.shape[0]), 0) + self.set_observations(x_train, uvi_train, self.x_abs_train, y_train, self.y_abs_train) + def calc_laplace(self, loghyp, f=None): self.kern.lengthscale = np.exp(loghyp[0:self._xdim]) @@ -265,7 +289,7 @@ def calc_laplace(self, loghyp, f=None): # With current hyperparameters: self.Kxx = self.kern.K(self.x_train_all) - self.iK, logdetK = self._safe_invert_noise(self.Kxx) + self.iK, self.logdetK = self._safe_invert_noise(self.Kxx) # First, solve for \hat{f} and W (mode finding Laplace approximation, Newton-Raphson) f_error = self.delta_f + 1 @@ -318,6 +342,7 @@ def calc_laplace(self, loghyp, f=None): self.W = W self.f = f self.iKf = np.matmul(self.iK, self.f) + self.KWI = np.matmul(self.W, self.Kxx) + self.Ix return f#, lml @@ -338,7 +363,12 @@ def _safe_invert_noise(self, mat): return imat, logdet def calc_nlml(self, loghyp): - f,lml = self.calc_laplace(loghyp) + f = self.calc_laplace(loghyp) + # Now calculate the log likelihoods (remember log(ax) = log a + log x) + log_py_f_rel = self.rel_likelihood.log_likelihood(self.uvi_train, self.y_train, f) + log_py_f_abs = self.abs_likelihood.log_likelihood(self.y_abs_train, f[self._n_rel:]) #TODO: I would prefer to use the indexing in absolute ratings too for consistency + fiKf = np.matmul(f.T, self.iKf) + lml = log_py_f_rel.sum()+log_py_f_abs.sum() - 0.5*fiKf - 0.5*np.log(np.linalg.det(self.KWI)) return -lml def predict_latent(self, x): @@ -346,16 +376,22 @@ def predict_latent(self, x): kt = self.kern.K(self.x_train_all, x) mean_latent = np.matmul(kt.T, self.iKf) Ktt = self.kern.K(x) - iKW = np.linalg.inv(np.matmul(self.W, self.Kxx) + self.Ix) + iKW = np.linalg.inv(self.KWI) var_latent = Ktt - np.matmul(kt.T, np.matmul(iKW, np.matmul(self.W, kt))) return mean_latent, var_latent + def predict_relative(self, x, uvi, fhat=None, varhat=None): + if fhat is None or varhat is None: + fhat, varhat = self.predict_latent(x) + p_y = self.rel_likelihood.prediction(fhat, varhat) + return p_y + def expected_y(self, x, fhat=None, varhat=None): if fhat is None or varhat is None: fhat, varhat = self.predict_latent(x) #Ktt = self.kern.K(x) #var_star = 2*self.abs_likelihood.sigma**2 + np.atleast_2d(Ktt.diagonal()).T - E_y = self.abs_likelihood.expectation(fhat, np.atleast_2d(varhat.diagonal()).T) + E_y = self.abs_likelihood.prediction(fhat, np.atleast_2d(varhat.diagonal()).T) return E_y diff --git a/active_learners.py b/active_learners.py new file mode 100644 index 0000000..4592913 --- /dev/null +++ b/active_learners.py @@ -0,0 +1,113 @@ +import numpy as np +import GPpref +from scipy.stats import beta + +class ActiveLearner(object): + def __init__(self, x_train, uvi_train, x_abs_train, y_train, y_abs_train, delta_f, abs_likelihood): + + self.nx_dim = x_abs_train.shape[1] + self.GP = GPpref.PreferenceGaussianProcess(x_train, uvi_train, x_abs_train, y_train, y_abs_train, + delta_f=delta_f, + abs_likelihood=abs_likelihood) + self.add_observations = self.GP.add_observations + + def set_hyperparameters(self, log_hyp): + self.log_hyp = log_hyp + + def calc_laplace(self): + self.f = self.GP.calc_laplace(self.log_hyp) + return self.f + + def predict_latent(self, x_test): + fhat, vhat = self.GP.predict_latent(x_test) + vhat = np.atleast_2d(vhat.diagonal()).T + return fhat, vhat + + def expected_y(self, x_test, fhat, vhat): + # Expected values0 + return self.GP.expected_y(x_test, fhat, vhat) + + def posterior_likelihood(self, fhat, vhat, y_samples, mc_samples): + # Sampling from posterior to generate output pdfs + p_y = np.zeros((len(y_samples), len(fhat))) + iny = 1.0/len(y_samples) + for i,(fstar,vstar) in enumerate(zip(fhat, vhat)): + f_samples = mc_samples*vstar+fstar + aa, bb = self.GP.abs_likelihood.get_alpha_beta(f_samples) + p_y[:, i] = [iny*np.sum(beta.pdf(yj, aa, bb)) for yj in y_samples] + # p_y[:, i] /= np.sum(p_y[:, i]) + return p_y + + def get_observations(self): + return self.GP.x_train, self.GP.uvi_train, self.GP.x_abs_train, self.GP.y_train, self.GP.y_abs_train + + def select_observation(self, domain=[0.0, 1.0]): + if np.random.uniform() < 0.5: + return np.random.uniform(low=domain[0], high=domain[1], size=(1,1)) + else: + return np.random.uniform(low=domain[0], high=domain[1], size=(2,1)) + + #def init_plots(self): + + + +class UCBLatent(ActiveLearner): + def select_observation(self, domain=[0.0, 1.0], ntest=100, gamma=2.0): + x_test = np.random.uniform(low=domain[0], high=domain[1], size=(ntest, 1)) + fhat, vhat = self.predict_latent(x_test) + return x_test[np.argmax(fhat + gamma*np.sqrt(vhat))] + +class UCBOut(ActiveLearner): + def select_observation(self, domain=[0.0, 1.0], ntest=100, gamma=2.0): + # Don't know how to recover the second moment of the predictive distribution, so this isn't done + x_test = np.random.uniform(low=domain[0], high=domain[1], size=(ntest, 1)) + fhat, vhat = self.predict_latent(x_test) + Ey = self.expected_y(x_test, fhat, vhat) + return x_test[np.argmax(Ey)] + +class PeakComparitor(ActiveLearner): + + def test_observation(self, x, y, uv, x_test, gamma): + self.add_observations(x, y, uv) + f = self.calc_laplace() + fhat, vhat = self.predict_latent(x_test) + max_ucb = (fhat + gamma*np.sqrt(np.atleast_2d(vhat.diagonal()).T)).max() + return max_ucb + + + def select_observation(self, domain=[0.0, 1.0], ntest=50, gamma=2.0): + crx, cuv, cax, cry, cay = self.get_observations() + + x_test = np.random.uniform(low=domain[0], high=domain[1], size=(ntest, 1)) + fhat, vhat = self.GP.predict_latent(x_test) + max_x = np.argmax(fhat) + other_x = np.delete(np.arange(ntest), max_x) + uv = np.vstack( (max_x*np.ones(ntest-1, dtype='int'), other_x) ).T + + p_pref = self.GP.rel_likelihood.prediction(fhat, vhat, uv) + V = np.zeros(ntest-1) + x = np.zeros((2,1), dtype='float') + x[0] = x_test[max_x] + ypos = np.ones((1, 1),dtype='int') + ruv = np.array([[0, 1]]) + + # Now calculate the expected value for each observation pair + for i,uv1 in enumerate(other_x): + x[1] = x_test[uv1] + V[i] += p_pref[i]*self.test_observation(x, -1*ypos, ruv, x_test, gamma) + self.GP.set_observations(crx, cuv, cax, cry, cay) + V[i] += (1-p_pref[i])*self.test_observation(x, ypos, ruv, x_test, gamma) + self.GP.set_observations(crx, cuv, cax, cry, cay) + + best = np.argmax(V) + x[1] = x_test[uv[best,1]] + cV = gamma*np.sqrt(np.atleast_2d(vhat.diagonal())).T + fhat + cVmax = np.argmax(cV) + if cV[cVmax] > V.max(): + x = np.array([x_test[cVmax]]) + return x + + + + + diff --git a/plot_tools.py b/plot_tools.py index 3b29036..10d1b5b 100644 --- a/plot_tools.py +++ b/plot_tools.py @@ -5,14 +5,18 @@ def make_poly_array(x,y,sigma): nx = len(x) + sigma = np.atleast_2d(sigma) xy = np.zeros((2*nx, 2)) xy[:,0] = np.append(x, x[::-1]) - xy[:,1] = np.append(y-sigma, y[::-1]+sigma) + xy[:,1] = np.append(y-sigma, y[::-1]+sigma[::-1]) return xy def plot_with_bounds(ax, x, y, s, c=lines[0]): - h_patch = Polygon(make_poly_array(x, y, s), ec=c, fc=lighten(c, 3), alpha=0.5) + xy = make_poly_array(x, y, s) + h_patch = Polygon(xy, ec=c, fc=lighten(c, 3), alpha=0.5) h_fx, = ax.plot(x, y, lw=1.5, c=c) ax.add_patch(h_patch) - return h_fx, h_patch \ No newline at end of file + clim = ax.get_ylim() + ax.set_ylim(bottom = min(clim[0],xy[:,1].min()), top = max(clim[1], xy[:,1].max())) + return h_fx, h_patch diff --git a/pref_active_learning.py b/pref_active_learning.py index c9cf94c..15945ab 100644 --- a/pref_active_learning.py +++ b/pref_active_learning.py @@ -6,47 +6,67 @@ import plot_tools as ptt import mcmc from scipy.stats import beta - -plt.rc('font', **{'family': 'serif', 'sans-serif': ['Computer Modern Roman']}) +import active_learners +plt.rc('font',**{'family':'serif','sans-serif':['Computer Modern Roman']}) plt.rc('text', usetex=True) -log_hyp = np.log([0.1, 0.5, 0.1, 10.0]) # length_scale, sigma_f, sigma_probit, v_beta +log_hyp = np.log([0.1,0.5,0.1,10.0]) # length_scale, sigma_f, sigma_probit, v_beta np.random.seed(3) -n_rel_train = 0 -n_abs_train = 0 +n_rel_train = 2 +n_abs_train = 1 true_sigma = 0.05 delta_f = 1e-5 +beta_sigma = 0.5 +beta_v=20.0 + n_xplot = 101 n_mcsamples = 1000 n_ysamples = 101 +n_queries = 20 -# Define polynomial function to be modelled +# Define function to be modelled def true_function(x): - y = (np.sin(x * 2 * np.pi + np.pi / 4) + 1.25) / 2.5 - # y = np.sin(x*2.0*np.pi + np.pi/4.0) + #y = (np.sin(x*2*np.pi + np.pi/4) + 1.25)/2.5 + #y = np.sin(x*2.0*np.pi + np.pi/4.0) + y = 0.3*np.cos(6*np.pi*(x-0.5))*np.exp(-10*(x-0.5)**2) + 0.5 return y +class ObservationFunction(object): + def __init__(self, true_fun): + self.f = true_fun -# Define noisy observation function -def obs_function(x, sigma): + def generate_observations(self, x): + fx = self.f(x) + +# Gaussian noise observation function +def normal_obs_function(x): fx = true_function(x) - noise = np.random.normal(scale=sigma, size=x.shape) + noise = np.random.normal(scale=true_sigma, size=x.shape) return fx + noise +beta_obs = GPpref.AbsBoundProbit(sigma=beta_sigma, v=beta_v) -def noisy_preference_rank(uv, sigma): - fuv = obs_function(uv, sigma) - y = -1 * np.ones((fuv.shape[0], 1), dtype='int') - y[fuv[:, 1] > fuv[:, 0]] = 1 - return y, fuv +def beta_obs_function(x): + fx = true_function(x) + a, b = beta_obs.get_alpha_beta(fx) + z = [beta.rvs(aa, bb) for aa,bb in zip(a,b)] + return z + +# Set target observation function +obs_function = normal_obs_function +def noisy_preference_rank(uv): + fuv = obs_function(uv) + y = -1*np.ones((fuv.shape[0],1),dtype='int') + y[fuv[:,1] > fuv[:,0]] = 1 + return y, fuv # Main program # True function -x_plot = np.linspace(0.0, 1.0, n_xplot, dtype='float') +x_plot = np.linspace(0.0,1.0,n_xplot,dtype='float') y_plot = true_function(x_plot) x_test = np.atleast_2d(x_plot).T @@ -54,84 +74,73 @@ def noisy_preference_rank(uv, sigma): # are actually indexes into x, because it is easier computationally. You can # recover the actual u,v values using x[ui],x[vi] if n_rel_train > 0: - x_train = np.random.random((2 * n_rel_train, 1)) - uvi_train = np.random.choice(range(2 * n_rel_train), (n_rel_train, 2), replace=False) - uv_train = x_train[uvi_train][:, :, 0] + x_train = np.random.random((2*n_rel_train,1)) + uvi_train = np.random.choice(range(2*n_rel_train), (n_rel_train,2), replace=False) + uv_train = x_train[uvi_train][:,:,0] # Get noisy observations f(uv) and corresponding ranks y_train - y_train, fuv_train = noisy_preference_rank(uv_train, true_sigma) + y_train, fuv_train = noisy_preference_rank(uv_train) else: - x_train = np.zeros((0, 1)) - uvi_train = np.zeros((0, 2)) - uv_train = np.zeros((0, 2)) - y_train = np.zeros((0, 1)) - fuv_train = np.zeros((0, 2)) - -x_abs_train = np.random.random((n_abs_train, 1)) -# y_abs_train = obs_function(x_abs_train, true_sigma) -y_abs_train = np.clip(obs_function(x_abs_train, true_sigma), 0.01, .99) - -# GP hyperparameters -# note: using log scaling to aid learning hyperparameters with varied magnitudes - -print "Data" -# print x_train -print x_abs_train -# print y_train -print y_abs_train -# print uvi_train - -prefGP = GPpref.PreferenceGaussianProcess(x_train, uvi_train, x_abs_train, y_train, y_abs_train, delta_f=delta_f) - -# Pseudocode: -# FOr a set of hyperparameters, return log likelihood that can be used by an optimiser + x_train = np.zeros((0,1)) + uvi_train = np.zeros((0,2)) + uv_train = np.zeros((0,2)) + y_train = np.zeros((0,1)) + fuv_train = np.zeros((0,2)) + +x_abs_train = np.random.random((n_abs_train,1)) +#y_abs_train = obs_function(x_abs_train, true_sigma) +y_abs_train = np.clip(obs_function(x_abs_train), 0.01, .99) + + +learner = active_learners.PeakComparitor(x_train, uvi_train, x_abs_train, y_train, y_abs_train, delta_f=delta_f, + abs_likelihood=GPpref.AbsBoundProbit(sigma=beta_sigma, v=beta_v)) + theta0 = log_hyp -# log_hyp = op.fmin(prefGP.calc_nlml,theta0) -# f,lml = prefGP.calc_laplace(log_hyp) -f = prefGP.calc_laplace(log_hyp) +# Get initial solution +learner.set_hyperparameters(log_hyp) +f = learner.calc_laplace() + +for obs_num in range(n_queries): + fuv = np.array([[0, 1]]) + next_x = np.atleast_2d(learner.select_observation()) + if next_x.shape[0] == 1: + next_y = obs_function(next_x) + learner.add_observations(next_x, next_y) + else: + next_y, next_f = noisy_preference_rank(next_x.T) + fuv_train = np.concatenate((fuv_train, next_f), 0) + learner.add_observations(next_x, next_y, fuv) + print next_x, next_y + f = learner.calc_laplace() # Latent predictions -fhat, vhat = prefGP.predict_latent(x_test) -vhat = np.atleast_2d(vhat.diagonal()).T +fhat, vhat = learner.predict_latent(x_test) # Expected values -E_y = prefGP.expected_y(x_test, fhat, vhat) +E_y = learner.expected_y(x_test, fhat, vhat) # Sampling from posterior to show likelihoods -p_y = np.zeros((n_ysamples, n_xplot)) -y_samples = np.linspace(0.0, 1.0, n_ysamples) -iny = 1.0 / n_ysamples -E_y2 = np.zeros(n_xplot) - -normal_samples = np.random.normal(size=n_mcsamples) -for i, (fstar, vstar) in enumerate(zip(fhat, vhat)): - f_samples = normal_samples * vstar + fstar - aa, bb = prefGP.abs_likelihood.get_alpha_beta(f_samples) - p_y[:, i] = [iny * np.sum(beta.pdf(yj, aa, bb)) for yj in y_samples] - E_y2[i] = np.sum(np.dot(y_samples, p_y[:, i])) / np.sum(p_y[:, i]) - -# New y's are expectations from Beta distribution. E(X) = alpha/(alpha+beta) -# alph = prefGP.abs_likelihood.alpha(f) -# bet = prefGP.abs_likelihood.beta(f) -# Ey = alph/(alph+bet) - -hf, (ha, hb) = plt.subplots(1, 2) +mc_samples = np.random.normal(size=n_mcsamples) +y_samples = np.linspace(0.01, 0.99, n_ysamples) +p_y = learner.posterior_likelihood(fhat, vhat, y_samples, mc_samples) + +hf, (ha, hb) = plt.subplots(1,2) hf, hpf = ptt.plot_with_bounds(ha, x_plot, y_plot, true_sigma, c=ptt.lines[0]) +ha.imshow(p_y, origin='lower', extent=[x_plot[0], x_plot[-1], 0.01, 0.99]) -if x_train.shape[0] > 0: - for uv, fuv, y in zip(uv_train, fuv_train, y_train): +if learner.GP.x_train.shape[0]>0: + for uv,fuv,y in zip(learner.GP.x_train[learner.GP.uvi_train][:,:,0], fuv_train, learner.GP.y_train): ha.plot(uv, fuv, 'b-', color=ptt.lighten(ptt.lines[0])) - ha.plot(uv[(y + 1) / 2], fuv[(y + 1) / 2], '+', color=ptt.darken(ptt.lines[0], 1.5)) + ha.plot(uv[(y+1)/2],fuv[(y+1)/2],'+', color=ptt.darken(ptt.lines[0], 1.5)) -if x_abs_train.shape[0] > 0: - ha.plot(x_abs_train, y_abs_train, '+', color=ptt.lighten(ptt.lines[2])) +if learner.GP.x_abs_train.shape[0]>0: + ha.plot(learner.GP.x_abs_train, learner.GP.y_abs_train, 'k+') hfhat, hpfhat = ptt.plot_with_bounds(hb, x_test, fhat, np.sqrt(vhat), c=ptt.lines[1]) hEy, = ha.plot(x_plot, E_y, color=ptt.lines[3]) -ha.imshow(p_y, origin='lower', extent=[x_plot[0], x_plot[-1], 0.0, 1.0]) # ha.plot(x_plot, E_y2, color=ptt.lines[4]) hmap, = ha.plot(x_plot, y_samples[np.argmax(p_y, axis=0)], color='w') ha.set_title('Training data') @@ -139,7 +148,8 @@ def noisy_preference_rank(uv, sigma): ha.set_xlabel('$x$') hb.set_xlabel('$x$') hb.set_ylabel('$f(x)$') -ha.legend([hf, hEy, hmap], [r'$f(x)$', r'$\mathbb{E}_{p(y|\mathcal{Y})}\left[y\right]$', r'$y_{MAP} | \mathcal{Y}$']) + +ha.legend([hf, hEy, hmap], [r'$f(x)$', r'$E_{p(y|\mathcal{Y})}\left[y\right]$', r'$y_{MAP} | \mathcal{Y}$']) hb.legend([hfhat], [r'Latent function $\hat{f}(x)$']) plt.show() From 7c0635cfde44ff0ebeffef01ecfdbea069824bd5 Mon Sep 17 00:00:00 2001 From: Nicholas Lawrance Date: Wed, 29 Mar 2017 16:18:59 -0700 Subject: [PATCH 09/38] Added bunch of plotting, sorted out likelihoods, training hypers works (with enough data) --- GP_preference_demo.py | 214 ++++++++++++++++----------------- GPpref.py | 259 +++++++++++++++++++++++++--------------- active_learners.py | 9 +- plot_tools.py | 31 +++++ pref_active_learning.py | 46 ++++--- 5 files changed, 330 insertions(+), 229 deletions(-) diff --git a/GP_preference_demo.py b/GP_preference_demo.py index 02582cf..36323a9 100644 --- a/GP_preference_demo.py +++ b/GP_preference_demo.py @@ -4,66 +4,42 @@ import GPpref import scipy.optimize as op import plot_tools as ptt -import mcmc -from scipy.stats import beta +# from scipy.stats import beta plt.rc('font',**{'family':'serif','sans-serif':['Computer Modern Roman']}) plt.rc('text', usetex=True) -log_hyp = np.log([0.1,0.5,0.1,10.0]) # length_scale, sigma_f, sigma_probit, v_beta -np.random.seed(3) +train_hyper = True -n_rel_train = 10 -n_abs_train = 10 -true_sigma = 0.05 +log_hyp = np.log([0.2, 0.5, 0.1, 1.0, 10.0]) # length_scale/s, sigma_f, sigma_n_abs, sigma_beta, v_beta +np.random.seed(1) + +n_rel_train = 30 +n_abs_train = 30 +rel_sigma = 0.2 delta_f = 1e-5 +beta_sigma = 0.8 +beta_v = 20.0 + n_xplot = 101 n_mcsamples = 1000 n_ysamples = 101 +marker_options = {'mec':'k', 'mew':0.5} # Define polynomial function to be modelled def true_function(x): - y = (np.sin(x*2*np.pi + np.pi/4) + 1.25)/2.5 + y = np.cos(6 * np.pi * (x - 0.5)) * np.exp(-10 * (x - 0.5) ** 2) + #y = (np.sin(x*2*np.pi + np.pi/4))/1.2 #y = np.sin(x*2.0*np.pi + np.pi/4.0) return y -class ObservationFunction(object): - def __init__(self, true_fun): - self.f = true_fun - - def generate_observations(self, x): - fx = self.f(x) - - - -# Gaussian noise observation function -def normal_obs_function(x): - fx = true_function(x) - noise = np.random.normal(scale=true_sigma, size=x.shape) - return fx + noise - -beta_obs = GPpref.AbsBoundProbit(sigma=true_sigma, v=10.0) - -def beta_obs_function(x): - fx = true_function(x) - a, b = beta_obs.get_alpha_beta(fx) - z = [beta.rvs(aa, bb) for aa,bb in zip(a,b)] - return z - -# Set target observation function -obs_function = normal_obs_function - -def noisy_preference_rank(uv): - fuv = obs_function(uv) - y = -1*np.ones((fuv.shape[0],1),dtype='int') - y[fuv[:,1] > fuv[:,0]] = 1 - return y, fuv +rel_obs_fun = GPpref.RelObservationSampler(true_function, GPpref.PrefProbit(sigma=rel_sigma)) +abs_obs_fun = GPpref.AbsObservationSampler(true_function, GPpref.AbsBoundProbit(sigma=beta_sigma, v=beta_v)) # Main program # True function x_plot = np.linspace(0.0,1.0,n_xplot,dtype='float') -y_plot = true_function(x_plot) x_test = np.atleast_2d(x_plot).T # Training data - this is a bit weird, but we sample x values, then the uv pairs @@ -74,8 +50,8 @@ def noisy_preference_rank(uv): uvi_train = np.random.choice(range(2*n_rel_train), (n_rel_train,2), replace=False) uv_train = x_train[uvi_train][:,:,0] - # Get noisy observations f(uv) and corresponding ranks y_train - y_train, fuv_train = noisy_preference_rank(uv_train) + # Get labels (y), and noisy observations f(uv) and corresponding ranks y_train + y_train, fuv_train = rel_obs_fun.generate_observations(uv_train) else: x_train = np.zeros((0,1)) @@ -84,81 +60,105 @@ def noisy_preference_rank(uv): y_train = np.zeros((0,1)) fuv_train = np.zeros((0,2)) +# For absolute points, get observation y and beta mean mu x_abs_train = np.random.random((n_abs_train,1)) -#y_abs_train = obs_function(x_abs_train, true_sigma) -y_abs_train = np.clip(obs_function(x_abs_train), 0.01, .99) - +y_abs_train, mu_abs_train = abs_obs_fun.generate_observations(x_abs_train) +prefGP = GPpref.PreferenceGaussianProcess(x_train, uvi_train, x_abs_train, y_train, y_abs_train, + delta_f=delta_f, + rel_likelihood=GPpref.PrefProbit(), + abs_likelihood=GPpref.AbsBoundProbit()) -# GP hyperparameters -# note: using log scaling to aid learning hyperparameters with varied magnitudes +# If training hyperparameters, use external optimiser +if train_hyper: + log_hyp = op.fmin(prefGP.calc_nlml,log_hyp) -print "Data" -# print x_train -print x_abs_train -# print y_train -print y_abs_train -# print uvi_train - -prefGP = GPpref.PreferenceGaussianProcess(x_train, uvi_train, x_abs_train, y_train, y_abs_train, delta_f=delta_f,abs_likelihood=GPpref.AbsBoundProbit(sigma=0.5, v=20.0)) - -# Pseudocode: -# FOr a set of hyperparameters, return log likelihood that can be used by an optimiser -theta0 = log_hyp - -log_hyp = op.fmin(prefGP.calc_nlml,theta0) -#f,lml = prefGP.calc_laplace(log_hyp) f = prefGP.calc_laplace(log_hyp) +prefGP.print_hyperparameters() # Latent predictions fhat, vhat = prefGP.predict_latent(x_test) -vhat = np.atleast_2d(vhat.diagonal()).T # Expected values -E_y = prefGP.expected_y(x_test, fhat, vhat) - -# Sampling from posterior to show likelihoods -p_y = np.zeros((n_ysamples, n_xplot)) -y_samples = np.linspace(0.01, 0.99, n_ysamples) -iny = 1.0/n_ysamples -E_y2 = np.zeros(n_xplot) - -normal_samples = np.random.normal(size=n_mcsamples) -for i,(fstar,vstar) in enumerate(zip(fhat, vhat)): - f_samples = normal_samples*vstar+fstar - aa, bb = prefGP.abs_likelihood.get_alpha_beta(f_samples) - p_y[:, i] = [iny*np.sum(beta.pdf(yj, aa, bb)) for yj in y_samples] - p_y[:, i] /= np.sum(p_y[:, i]) - E_y2[i] = np.sum(np.dot(y_samples, p_y[:, i])) - -# New y's are expectations from Beta distribution. E(X) = alpha/(alpha+beta) -#alph = prefGP.abs_likelihood.alpha(f) -#bet = prefGP.abs_likelihood.beta(f) -#Ey = alph/(alph+bet) - -hf, (ha, hb) = plt.subplots(1,2) -hf, hpf = ptt.plot_with_bounds(ha, x_plot, y_plot, true_sigma, c=ptt.lines[0]) - -if x_train.shape[0]>0: - for uv,fuv,y in zip(uv_train, fuv_train, y_train): - ha.plot(uv, fuv, 'b-', color=ptt.lighten(ptt.lines[0])) - ha.plot(uv[(y+1)/2],fuv[(y+1)/2],'+', color=ptt.darken(ptt.lines[0], 1.5)) +E_y = prefGP.abs_posterior_mean(x_test, fhat, vhat) -if x_abs_train.shape[0]>0: - ha.plot(x_abs_train, y_abs_train, '+', color=ptt.lighten(ptt.lines[2])) - -hfhat, hpfhat = ptt.plot_with_bounds(hb, x_test, fhat, np.sqrt(vhat), c=ptt.lines[1]) -hEy, = ha.plot(x_plot, E_y, color=ptt.lines[3]) - -ha.imshow(p_y, origin='lower', extent=[x_plot[0], x_plot[-1], 0.01, 0.99]) -# ha.plot(x_plot, E_y2, color=ptt.lines[4]) -hmap, = ha.plot(x_plot, y_samples[np.argmax(p_y, axis=0)], color='w') -ha.set_title('Training data') -ha.set_ylabel('$y$') -ha.set_xlabel('$x$') -hb.set_xlabel('$x$') -hb.set_ylabel('$f(x)$') -ha.legend([hf, hEy, hmap], [r'$f(x)$', r'$\mathbb{E}_{p(y|\mathcal{Y})}\left[y\right]$', r'$y_{MAP} | \mathcal{Y}$']) -hb.legend([hfhat], [r'Latent function $\hat{f}(x)$']) +# Absolute posterior likelihood (MC sampled) +mc_samples = np.random.normal(size=n_mcsamples) +abs_y_samples = np.atleast_2d(np.linspace(0.01, 0.99, n_ysamples)).T +p_y = prefGP.abs_posterior_likelihood(abs_y_samples, fhat=fhat, varhat=vhat, normal_samples=mc_samples) + +# PLOTTING + +# Plot true function, likelihoods and observations +fig_t, (ax_t_l, ax_t_a, ax_t_r) = ptt.plot_setup_2d(t_l=r'True latent function, $f(x)$') +# True latent +f_true = abs_obs_fun.f(x_test) +ptt.plot_with_bounds(ax_t_l, x_test, f_true, rel_sigma, c=ptt.lines[0]) + +# True absolute likelihood +p_abs_y_true = abs_obs_fun.observation_likelihood_array(x_test, abs_y_samples) +abs_extent = [x_test[0,0], x_test[-1,0], abs_y_samples[0,0], abs_y_samples[-1,0]] +h_pat = ax_t_a.imshow(p_abs_y_true, origin='lower', extent=abs_extent) +if x_abs_train.shape[0]>0: + ax_t_a.plot(x_abs_train, y_abs_train, 'w+') +mu_true = abs_obs_fun.mean_link(x_test) +h_yt, = ax_t_a.plot(x_test, mu_true, c=ptt.lines[0]) +ax_t_a.legend([h_yt], ['$E(y|f(x))$']) +fig_t.colorbar(h_pat, ax=ax_t_a) + +# True relative likelihood +p_rel_y_true = rel_obs_fun.observation_likelihood_array(x_test) +rel_y_extent = [x_test.min(), x_test.max(), x_test.min(), x_test.max()] +h_prt = ptt.plot_relative_likelihood(ax_t_r, p_rel_y_true, extent=rel_y_extent) +class_icons = ['ko','wo'] +if x_train.shape[0] > 0: + for uv, fuv, y in zip(uv_train, fuv_train, y_train): + ax_t_r.plot(uv[0], uv[1], class_icons[(y[0]+1)/2], **marker_options) + ax_t_l.plot(uv, fuv, 'b-', color=ptt.lighten(ptt.lines[0])) + ax_t_l.plot(uv[(y+1)/2], fuv[(y+1)/2], class_icons[(y[0]+1)/2], **marker_options) # '+', color=ptt.darken(ptt.lines[0], 1.5) + +# Posterior estimates +fig_p, (ax_p_l, ax_p_a, ax_p_r) = ptt.plot_setup_2d( + t_a=r'Posterior absolute likelihood, $p(y | \mathcal{Y}, \theta)$', + t_r=r'Posterior relative likelihood $P(x_0 \succ x_1 | \mathcal{Y}, \theta)$') + +# Latent function +hf, hpf = ptt.plot_with_bounds(ax_p_l, x_test, f_true, rel_sigma, c=ptt.lines[0]) + +hf_hat, hpf_hat = ptt.plot_with_bounds(ax_p_l, x_test, fhat, np.sqrt(np.atleast_2d(vhat.diagonal()).T), c=ptt.lines[1]) +ax_p_l.legend([hf, hf_hat], [r'True latent function, $f(x)$', r'$\mathcal{GP}$ estimate $\hat{f}(x)$']) + +# Absolute posterior likelihood +h_pap = ax_p_a.imshow(p_y, origin='lower', extent=abs_extent) +h_yt, = ax_p_a.plot(x_test, mu_true, c=ptt.lines[0]) +hEy, = ax_p_a.plot(x_plot, E_y, color=ptt.lines[3]) +if x_abs_train.shape[0]>0: + ax_p_a.plot(x_abs_train, y_abs_train, 'w+') +ax_p_a.legend([h_yt, hEy], [r'True mean, $E_{p(y|f(x))}[y]$', r'Posterior mean, $E_{p(y|\mathcal{Y})}\left[y\right]$']) +fig_p.colorbar(h_pap, ax=ax_p_a) + +# Relative posterior likelihood +p_rel_y_post = prefGP.rel_posterior_likelihood_array(fhat=fhat, varhat=vhat) +h_prp = ptt.plot_relative_likelihood(ax_p_r, p_rel_y_post, extent=rel_y_extent) +if x_train.shape[0] > 0: + for uv, fuv, y in zip(uv_train, fuv_train, y_train): + ax_p_r.plot(uv[0], uv[1], class_icons[(y[0]+1)/2], **marker_options) + ax_p_l.plot(uv, fuv, 'b-', color=ptt.lighten(ptt.lines[0])) + ax_p_l.plot(uv[(y+1)/2], fuv[(y+1)/2], class_icons[(y[0]+1)/2], **marker_options) # '+', color=ptt.darken(ptt.lines[0], 1.5) plt.show() + + +## SCRAP +# p_y = np.zeros((n_ysamples, n_xplot)) +# y_samples = np.linspace(0.01, 0.99, n_ysamples) +# iny = 1.0/n_ysamples +# E_y2 = np.zeros(n_xplot) +# +# normal_samples = np.random.normal(size=n_mcsamples) +# for i,(fstar,vstar) in enumerate(zip(fhat, vhat.diagonal())): +# f_samples = normal_samples*vstar+fstar +# aa, bb = prefGP.abs_likelihood.get_alpha_beta(f_samples) +# p_y[:, i] = [iny*np.sum(beta.pdf(yj, aa, bb)) for yj in y_samples] +# p_y[:, i] /= np.sum(p_y[:, i]) +# E_y2[i] = np.sum(np.dot(y_samples, p_y[:, i])) \ No newline at end of file diff --git a/GPpref.py b/GPpref.py index 926e00c..2a8b2aa 100644 --- a/GPpref.py +++ b/GPpref.py @@ -14,7 +14,7 @@ def std_norm_pdf(x): #return np.exp(-(x**2)/2)/_sqrt_2pi def std_norm_cdf(x): - x = np.clip(x, -30, 100 ) + #x = np.clip(x, -30, 100 ) return norm.cdf(x) # Define squared distance calculation function @@ -29,6 +29,13 @@ def squared_distance(A,B): sqDist = A2 + B2 - AB return sqDist +def print_hyperparameters(theta, log=False): + if log is True: + theta = np.exp(theta) + nl = len(theta)-3 + lstr = ', '.join(['%.2f']*nl) % tuple(theta[:nl]) + print "l: {0}, sig_f: {1:0.2f}, sig: {2:0.2f}, v: {3:0.2f}".format(lstr, theta[-3], theta[-2], theta[-1]) + # Define squared exponential CovarianceFunction function class SquaredExponential(object): def __init__(self, logHyp, x): @@ -60,9 +67,15 @@ def set_sigma(self, sigma): self.sigma = sigma self._isqrt2sig = 1.0 / (self.sigma * np.sqrt(2.0)) self._i2var = self._isqrt2sig**2 - - def z_k(self, uvi, y, f): - zc = self._isqrt2sig * (f[uvi[:, 1]] - f[uvi[:, 0]]) + + def print_hyperparameters(self): + print "Probit relative, Gaussian noise on latent. ", + print "Sigma: {0:0.2f}".format(self.sigma) + + def z_k(self, y, f, scale=None): + if scale is None: + scale = self._isqrt2sig + zc = scale * (f[:, 1, None] - f[:, 0, None]) # Weird none is to preserve shape return y * zc def I_k(self, x, uv): # Jensen and Nielsen @@ -75,7 +88,7 @@ def I_k(self, x, uv): # Jensen and Nielsen def derivatives(self, uvi, y, f): nx = len(f) - z = self.z_k(uvi, y=y, f=f) + z = self.z_k(y=y, f=self.get_rel_f(f, uvi)) phi_z = std_norm_cdf(z) N_z = std_norm_pdf(z) @@ -96,22 +109,34 @@ def derivatives(self, uvi, y, f): return W, dpy_df - def likelihood(self, uvi, y, f): - z = self.z_k(uvi, y=y, f=f) + def get_rel_f(self, f, uvi): + return np.hstack((f[uvi[:, 0]], f[uvi[:, 1]])) + + def likelihood(self, y, f, scale=None): + z = self.z_k(y, f, scale=scale) phi_z = std_norm_cdf(z) return phi_z - def log_likelihood(self, uvi, y, f): - return np.log(self.likelihood(uvi, y, f)) + def log_likelihood(self, y, f): + return np.log(self.likelihood(y, f)) - def prediction(self, fhat, varhat, uvi): + def posterior_likelihood(self, fhat, varhat, uvi, y=1): # This is the likelihood assuming a Gaussian over f var_star = 2*self.sigma**2 + np.atleast_2d([varhat[u, u] + varhat[v, v] - varhat[u, v] - varhat[v, v] for u,v in uvi]).T - p_y = std_norm_cdf( (fhat[uvi[:,0]] - fhat[uvi[:,1]])/np.sqrt(var_star) ) + p_y = self.likelihood(y, self.get_rel_f(fhat, uvi), 1.0/np.sqrt(var_star)) return p_y + def generate_samples(self, f): + fuv = f + np.random.normal(scale=self.sigma, size=f.shape) + y = -1 * np.ones((fuv.shape[0], 1), dtype='int') + y[fuv[:, 1] > fuv[:, 0]] = 1 + return y, fuv + class AbsBoundProbit(object): def __init__(self, sigma=1.0, v=10.0): - # Making v=10 looks good on a graph, but I'm not sure what it's actually supposed to be. + # v is the precision, kind of related to inverse of noise, high v is sharp distributions + # sigma is the slope of the probit, basically scales how far away from + # 0 the latent has to be to to move away from 0.5 output. Sigma should + # basically relate to the range of the latent function self.set_sigma(sigma) self.set_v(v) self.log2pi = np.log(2.0*np.pi) @@ -123,6 +148,10 @@ def set_sigma(self, sigma): def set_v(self, v): self.v = v + + def print_hyperparameters(self): + print "Beta distribution, probit mean link.", + print "Sigma: {0:0.2f}, v: {1:0.2f}".format(self.sigma, self.v) def mean_link(self, f): ml = np.clip(std_norm_cdf(f*self._isqrt2sig), 1e-12, 1.0-1e-12) @@ -149,99 +178,50 @@ def log_likelihood(self, y, f): def derivatives(self, y, f): - #print "Start Iter. f, y" - #print f - #print y alpha = self.alpha(f) beta = self.beta(f) #let's make a distribution called beta that also has beta as a parameter! - # print "Alpha: " + str(alpha) - # print "Beta: " + str(beta) - # Estimate dpy_df - delta = 0.001 - #print "Estimated dpy_df" - #est_dpy_df = (self.log_likelihood(y, f+delta) - self.log_likelihood(y, f-delta))/(2*delta) - #print est_dpy_df - #print 'log likelihood' - #print np.sum(self.log_likelihood(y, f)) - - #print "Estimated W" - #est_W_diag = (self.log_likelihood(y, f+2*delta) - 2*self.log_likelihood(y,f) + self.log_likelihood(y, f-2*delta))/(2*delta)**2 - - #print est_W_diag - - # Theres a dot in Jensen that I'm hoping is just a typo. I didn't fully derive it, but it looks correct. - # As the iteration goes, f blows up. This makes parts of alpha, beta go to v and 0. Digamma(0)=-inf :( - # So why is it blowing up? dpy_df = self.v*self._isqrt2sig*std_norm_pdf(f*self._isqrt2sig) * (np.log(y)-np.log(1-y) - digamma(alpha) + digamma(beta) ) - Wdiag = - self.v*self._isqrt2sig*std_norm_pdf(f*self._isqrt2sig) * ( + Wdiag = - self.v*self._isqrt2sig*std_norm_pdf(f*self._isqrt2sig) * ( f*self._i2var*( np.log(y)-np.log(1.0-y)-digamma(alpha) + digamma(beta) ) + self.v*self._isqrt2sig*std_norm_pdf(f*self._isqrt2sig) * (polygamma(1, alpha) + polygamma(1, beta)) ) - # print "Wdiag" - # print Wdiag W = np.diagflat(Wdiag) return -W, dpy_df - def log_marginal(self): - pass - - def prediction(self, fhat, var_star): + def posterior_mean(self, fhat, var_star): #mu_t = self.mean_link(fhat) E_x = np.clip(std_norm_cdf(fhat/(np.sqrt(2*self.sigma**2 + var_star))), 1e-12, 1.0-1e-12) return E_x -class AbsProbit(object): - # The Probit Likelihood given in Rasmussen, p43. Note f(x) is scaled by sqrt(2)*sigma, as in Jensen. - # Ok, yeah, this was totally wrong. Don't use this. - def __init__(self, sigma=1.0, v=10.0): - self.set_sigma(sigma) - self.log2pi = np.log(2.0*np.pi) - - def set_sigma(self, sigma): - self.sigma = sigma - self._isqrt2sig = 1.0 / (self.sigma * np.sqrt(2.0)) - self._i2var = self._isqrt2sig**2 - - def derivatives(self, y, f): - - print "Start Iter. f, y" - print f - print y - - # dpy_df = -y*std_norm_pdf(f*self._isqrt2sig) / std_norm_cdf(y*f*self._isqrt2sig) - - # Wdiag = -(-np.power(std_norm_pdf(f*self._isqrt2sig) / std_norm_cdf(y*f*self._isqrt2sig),2) - # -y*f*self._isqrt2sig*std_norm_pdf(f*self._isqrt2sig) / std_norm_cdf(y*f*self._isqrt2sig)) - - dpy_df = -y*std_norm_pdf(f) / std_norm_cdf(y*f) - - Wdiag = -(-np.power(std_norm_pdf(f) / std_norm_cdf(y*f),2) - -y*f*std_norm_pdf(f) / std_norm_cdf(y*f)) + def generate_samples(self, f): + mu = self.mean_link(f) + a, b = self.get_alpha_beta(f) + z = np.zeros(f.shape, dtype='float') + for i, (aa,bb) in enumerate(zip(a,b)): + z[i] = beta.rvs(aa, bb) + return z, mu - W = np.diagflat(Wdiag) - - return W, dpy_df - - class PreferenceGaussianProcess(object): - def __init__(self, x_train, uvi_train, x_abs_train, y_train, y_abs_train, likelihood=PrefProbit(), delta_f = 1e-6, abs_likelihood=AbsBoundProbit()): + def __init__(self, x_train, uvi_train, x_abs_train, y_train, y_abs_train, rel_likelihood=PrefProbit(), delta_f = 1e-6, abs_likelihood=AbsBoundProbit()): # log_hyp are log of hyperparameters, note that it is [length_0, ..., length_d, sigma_f, sigma_probit, v_beta] # Training points are split into relative and absolute for calculating f, but combined for predictions. self.set_observations(x_train, uvi_train, x_abs_train, y_train, y_abs_train) self.delta_f = delta_f - self.rel_likelihood = likelihood + self.rel_likelihood = rel_likelihood self.abs_likelihood = abs_likelihood self.kern = GPy.kern.RBF(self._xdim, ARD=True) self.Ix = np.eye(self._nx) + self.f = None + def set_observations(self, x_train, uvi_train, x_abs_train, y_train, y_abs_train): if x_train.shape[0] is not 0: self._xdim = x_train.shape[1] @@ -274,17 +254,17 @@ def add_observations(self, x, y, uvi=None): uvi_train = np.concatenate((self.uvi_train, uvi+self.x_train.shape[0]), 0) self.set_observations(x_train, uvi_train, self.x_abs_train, y_train, self.y_abs_train) - - def calc_laplace(self, loghyp, f=None): + def calc_laplace(self, loghyp): self.kern.lengthscale = np.exp(loghyp[0:self._xdim]) self.kern.variance = (np.exp(loghyp[self._xdim]))**2 - self.rel_likelihood.set_sigma = np.exp(loghyp[-2]) # Do we need different sigmas for each likelihood? Hopefully No? - self.abs_likelihood.set_sigma = np.exp(loghyp[-2]) - self.abs_likelihood.set_v = np.exp(loghyp[-1]) + self.rel_likelihood.set_sigma(np.exp(loghyp[-3])) # Do we need different sigmas for each likelihood? Yes! + self.abs_likelihood.set_sigma(np.exp(loghyp[-2])) # I think this sigma relates to sigma_f in the covariance, and is actually possibly redundant + self.abs_likelihood.set_v(np.exp(loghyp[-1])) # Should this relate to the rel_likelihood probit noise? - if f is None: - f = np.ones((self._nx, 1)) - f = f*.0 + if self.f is None or self.f.shape[0] is not self._nx: + f = np.zeros((self._nx, 1), dtype='float') + else: + f = self.f # With current hyperparameters: self.Kxx = self.kern.K(self.x_train_all) @@ -364,8 +344,10 @@ def _safe_invert_noise(self, mat): def calc_nlml(self, loghyp): f = self.calc_laplace(loghyp) + if self.f is None: + self.f = f # Now calculate the log likelihoods (remember log(ax) = log a + log x) - log_py_f_rel = self.rel_likelihood.log_likelihood(self.uvi_train, self.y_train, f) + log_py_f_rel = self.rel_likelihood.log_likelihood(self.y_train, self.rel_likelihood.get_rel_f(f, self.uvi_train)) log_py_f_abs = self.abs_likelihood.log_likelihood(self.y_abs_train, f[self._n_rel:]) #TODO: I would prefer to use the indexing in absolute ratings too for consistency fiKf = np.matmul(f.T, self.iKf) lml = log_py_f_rel.sum()+log_py_f_abs.sum() - 0.5*fiKf - 0.5*np.log(np.linalg.det(self.KWI)) @@ -380,21 +362,112 @@ def predict_latent(self, x): var_latent = Ktt - np.matmul(kt.T, np.matmul(iKW, np.matmul(self.W, kt))) return mean_latent, var_latent - def predict_relative(self, x, uvi, fhat=None, varhat=None): - if fhat is None or varhat is None: - fhat, varhat = self.predict_latent(x) - p_y = self.rel_likelihood.prediction(fhat, varhat) + def _check_latent_input(self, x=None, fhat=None, varhat=None): + if (fhat is None or varhat is None): + if x is not None: + fhat, varhat = self.predict_latent(x) + else: + raise ValueError('Must supply either x or fhat and varhat') + return fhat, varhat + + def rel_posterior_likelihood(self, uvi, y, x=None, fhat=None, varhat=None): + fhat, varhat = self._check_latent_input(x, fhat, varhat) + p_y = self.rel_likelihood.posterior_likelihood(fhat, varhat, uvi, y) + return p_y + + def rel_posterior_likelihood_array(self, x=None, fhat=None, varhat=None, y=-1): + fhat, varhat = self._check_latent_input(x, fhat, varhat) + p_y = np.zeros((fhat.shape[0], fhat.shape[0]), dtype='float') + cross_uvi = np.zeros((fhat.shape[0],2), dtype='int') + cross_uvi[:, 1] = np.arange(fhat.shape[0]) + for i in cross_uvi[:, 1]: + cross_uvi[:, 0] = i + p_y[:, i:i+1] = self.rel_likelihood.posterior_likelihood(fhat, varhat, cross_uvi, y) return p_y - def expected_y(self, x, fhat=None, varhat=None): - if fhat is None or varhat is None: - fhat, varhat = self.predict_latent(x) - #Ktt = self.kern.K(x) - #var_star = 2*self.abs_likelihood.sigma**2 + np.atleast_2d(Ktt.diagonal()).T - E_y = self.abs_likelihood.prediction(fhat, np.atleast_2d(varhat.diagonal()).T) + def rel_posterior_MAP(self, uvi, x=None, fhat=None, varhat=None): + p_y = self.rel_posterior_likelihood(uvi, y=1, x=x, fhat=fhat, varhat=varhat) + return 2*np.array(p_y < 0.5, dtype='int')-1 + + def abs_posterior_likelihood(self, y, x=None, fhat=None, varhat=None, normal_samples = None): # Currently sample-based + fhat, varhat = self._check_latent_input(x, fhat, varhat) + varhat = np.atleast_2d(varhat.diagonal()).T + if normal_samples is None: + normal_samples = np.random.normal(size=1000) + iny = 1.0/len(normal_samples) + + # Sampling from posterior to show likelihoods + p_y = np.zeros((y.shape[0], fhat.shape[0])) + for i, (fstar, vstar) in enumerate(zip(fhat, varhat)): + f_samples = normal_samples * vstar + fstar + p_y[:, i] = [iny * np.sum(self.abs_likelihood.likelihood(yj, f_samples)) for yj in y] + return p_y + + def abs_posterior_mean(self, x=None, fhat=None, varhat=None): + fhat, varhat = self._check_latent_input(x, fhat, varhat) + varstar = np.atleast_2d(varhat.diagonal()).T + E_y = self.abs_likelihood.posterior_mean(fhat, varstar) return E_y + def print_hyperparameters(self): + print "COV: '{0}', l: {1}, sigma_f: {2}".format(self.kern.name, self.kern.lengthscale.values, np.sqrt(self.kern.variance.values)) + print "REL: ", + self.rel_likelihood.print_hyperparameters() + print "ABS: ", + self.abs_likelihood.print_hyperparameters() + + +class ObservationSampler(object): + def __init__(self, true_fun, likelihood_object): + self.f = true_fun + self.l = likelihood_object + def generate_observations(self, x): + fx = self.f(x) + y, ff = self.l.generate_samples(fx) + return y, ff +class AbsObservationSampler(ObservationSampler): + def observation_likelihood_array(self, x, y): + fx = self.f(x) + p_y = np.zeros((y.shape[0], fx.shape[0]), dtype='float') + for i, fxi in enumerate(fx): + p_y[:, i:i + 1] = self.l.likelihood(y, fxi[0]) + return p_y + + def mean_link(self, x): + fx = self.f(x) + return self.l.mean_link(fx) + + +class RelObservationSampler(ObservationSampler): + def observation_likelihood_array(self, x, y=-1): + fx = self.f(x) + p_y = np.zeros((x.shape[0], x.shape[0]), dtype='float') + cross_fx = np.hstack((np.zeros((x.shape[0], 1)), fx)) + for i, fxi in enumerate(fx): + cross_fx[:, 0] = fxi + p_y[:, i:i + 1] = self.l.likelihood(y, cross_fx) + return p_y + + + + + + +# SCRAP: + # Estimate dpy_df + # delta = 0.001 + # print "Estimated dpy_df" + # est_dpy_df = (self.log_likelihood(y, f+delta) - self.log_likelihood(y, f-delta))/(2*delta) + # print est_dpy_df + # print 'log likelihood' + # print np.sum(self.log_likelihood(y, f)) + # + # print "Estimated W" + # est_W_diag = (self.log_likelihood(y, f+2*delta) - 2*self.log_likelihood(y,f) + self.log_likelihood(y, f-2*delta))/(2*delta)**2 + # + # print est_W_diag + diff --git a/active_learners.py b/active_learners.py index 4592913..bbfa9c9 100644 --- a/active_learners.py +++ b/active_learners.py @@ -27,7 +27,7 @@ def expected_y(self, x_test, fhat, vhat): # Expected values0 return self.GP.expected_y(x_test, fhat, vhat) - def posterior_likelihood(self, fhat, vhat, y_samples, mc_samples): + def abs_posterior_likelihood(self, fhat, vhat, y_samples, mc_samples): # Sampling from posterior to generate output pdfs p_y = np.zeros((len(y_samples), len(fhat))) iny = 1.0/len(y_samples) @@ -35,7 +35,6 @@ def posterior_likelihood(self, fhat, vhat, y_samples, mc_samples): f_samples = mc_samples*vstar+fstar aa, bb = self.GP.abs_likelihood.get_alpha_beta(f_samples) p_y[:, i] = [iny*np.sum(beta.pdf(yj, aa, bb)) for yj in y_samples] - # p_y[:, i] /= np.sum(p_y[:, i]) return p_y def get_observations(self): @@ -75,16 +74,16 @@ def test_observation(self, x, y, uv, x_test, gamma): return max_ucb - def select_observation(self, domain=[0.0, 1.0], ntest=50, gamma=2.0): + def select_observation(self, domain=[0.0, 1.0], ntest=50, gamma=2.0, n_comparitors=1): crx, cuv, cax, cry, cay = self.get_observations() x_test = np.random.uniform(low=domain[0], high=domain[1], size=(ntest, 1)) fhat, vhat = self.GP.predict_latent(x_test) max_x = np.argmax(fhat) other_x = np.delete(np.arange(ntest), max_x) - uv = np.vstack( (max_x*np.ones(ntest-1, dtype='int'), other_x) ).T + uv = np.vstack((max_x*np.ones(ntest-1, dtype='int'), other_x)).T - p_pref = self.GP.rel_likelihood.prediction(fhat, vhat, uv) + p_pref = self.GP.rel_likelihood.posterior_likelihood(fhat, vhat, uv, y=-1) V = np.zeros(ntest-1) x = np.zeros((2,1), dtype='float') x[0] = x_test[max_x] diff --git a/plot_tools.py b/plot_tools.py index 10d1b5b..785b342 100644 --- a/plot_tools.py +++ b/plot_tools.py @@ -20,3 +20,34 @@ def plot_with_bounds(ax, x, y, s, c=lines[0]): clim = ax.get_ylim() ax.set_ylim(bottom = min(clim[0],xy[:,1].min()), top = max(clim[1], xy[:,1].max())) return h_fx, h_patch + + +def plot_setup_2d(t_l = r'Latent function, $f(x)$', t_a = r'Absolute likelihood, $p(y | f(x))$', + t_r = r'Relative likelihood, $P(x_0 \succ x_1 | f(x_0), f(x_1))$'): + + fig, (ax_l, ax_a, ax_r) = plt.subplots(1, 3) + fig.set_size_inches(14.7, 3.5) + + # Latent function + ax_l.set_title(t_l) + ax_l.set_xlabel('$x$') + ax_l.set_ylabel('$f(x)$') + + # Absolute likelihood + ax_a.set_title(t_a) + ax_a.set_xlabel('$x$') + ax_a.set_ylabel('$y$') + + # Relative likelihood + ax_r.set_title(t_r) + ax_r.set_xlabel('$x_0$') + ax_r.set_ylabel('$x_1$') + return fig, (ax_l, ax_a, ax_r) + + +def plot_relative_likelihood(ax, p_y, extent): + h_p = ax.imshow(p_y, origin='lower', extent=extent, vmin=0.0, vmax=1.0) + h_pc = ax.contour(p_y, levels=[0.5], origin='lower', linewidths=2, extent=extent) + plt.clabel(h_pc, inline=1, fontsize=10) + ax.get_figure().colorbar(h_p, ax=ax) + return h_p \ No newline at end of file diff --git a/pref_active_learning.py b/pref_active_learning.py index 15945ab..03937d0 100644 --- a/pref_active_learning.py +++ b/pref_active_learning.py @@ -10,16 +10,17 @@ plt.rc('font',**{'family':'serif','sans-serif':['Computer Modern Roman']}) plt.rc('text', usetex=True) -log_hyp = np.log([0.1,0.5,0.1,10.0]) # length_scale, sigma_f, sigma_probit, v_beta +# log_hyp = np.log([0.1,0.5,0.1,10.0]) # length_scale, sigma_f, sigma_probit, v_beta +log_hyp = np.log([0.07, 0.6, 0.25, 1.0, 28.1]) np.random.seed(3) n_rel_train = 2 n_abs_train = 1 -true_sigma = 0.05 +rel_sigma = 0.05 delta_f = 1e-5 -beta_sigma = 0.5 -beta_v=20.0 +beta_sigma = 0.8 +beta_v = 20.0 n_xplot = 101 n_mcsamples = 1000 @@ -31,35 +32,29 @@ def true_function(x): #y = (np.sin(x*2*np.pi + np.pi/4) + 1.25)/2.5 #y = np.sin(x*2.0*np.pi + np.pi/4.0) - y = 0.3*np.cos(6*np.pi*(x-0.5))*np.exp(-10*(x-0.5)**2) + 0.5 + y = np.cos(6 * np.pi * (x - 0.5)) * np.exp(-10 * (x - 0.5) ** 2) return y -class ObservationFunction(object): - def __init__(self, true_fun): - self.f = true_fun - - def generate_observations(self, x): - fx = self.f(x) +rel_obs_fun = GPpref.RelObservationSampler(true_function, GPpref.PrefProbit(sigma=rel_sigma)) +abs_obs_fun = GPpref.AbsObservationSampler(true_function, GPpref.AbsBoundProbit(sigma=beta_sigma, v=beta_v)) # Gaussian noise observation function def normal_obs_function(x): fx = true_function(x) - noise = np.random.normal(scale=true_sigma, size=x.shape) + noise = np.random.normal(scale=rel_sigma, size=x.shape) return fx + noise beta_obs = GPpref.AbsBoundProbit(sigma=beta_sigma, v=beta_v) def beta_obs_function(x): fx = true_function(x) - a, b = beta_obs.get_alpha_beta(fx) + mu = GPpref.mean_link(f) + a, b = beta_obs.get_alpha_beta(mu) z = [beta.rvs(aa, bb) for aa,bb in zip(a,b)] return z -# Set target observation function -obs_function = normal_obs_function - def noisy_preference_rank(uv): - fuv = obs_function(uv) + fuv = normal_obs_function(uv) y = -1*np.ones((fuv.shape[0],1),dtype='int') y[fuv[:,1] > fuv[:,0]] = 1 return y, fuv @@ -89,8 +84,8 @@ def noisy_preference_rank(uv): fuv_train = np.zeros((0,2)) x_abs_train = np.random.random((n_abs_train,1)) -#y_abs_train = obs_function(x_abs_train, true_sigma) -y_abs_train = np.clip(obs_function(x_abs_train), 0.01, .99) +y_abs_train = beta_obs_function(x_abs_train, sigma=rel_sigma) +#y_abs_train = np.clip(normal_obs_function(x_abs_train), 0.01, .99) learner = active_learners.PeakComparitor(x_train, uvi_train, x_abs_train, y_train, y_abs_train, delta_f=delta_f, @@ -106,7 +101,7 @@ def noisy_preference_rank(uv): fuv = np.array([[0, 1]]) next_x = np.atleast_2d(learner.select_observation()) if next_x.shape[0] == 1: - next_y = obs_function(next_x) + next_y = beta_obs_function(next_x, sigma=rel_sigma) learner.add_observations(next_x, next_y) else: next_y, next_f = noisy_preference_rank(next_x.T) @@ -119,15 +114,18 @@ def noisy_preference_rank(uv): fhat, vhat = learner.predict_latent(x_test) # Expected values -E_y = learner.expected_y(x_test, fhat, vhat) +E_y = learner.GP.abs_posterior_mean(x_test, fhat, vhat) # Sampling from posterior to show likelihoods mc_samples = np.random.normal(size=n_mcsamples) y_samples = np.linspace(0.01, 0.99, n_ysamples) -p_y = learner.posterior_likelihood(fhat, vhat, y_samples, mc_samples) +p_y = learner.GP.abs_posterior_likelihood(y_samples, fhat=fhat, varhat=vhat, normal_samples=mc_samples) + +hf_input, ha_input =plt.subplot(1,1) + -hf, (ha, hb) = plt.subplots(1,2) -hf, hpf = ptt.plot_with_bounds(ha, x_plot, y_plot, true_sigma, c=ptt.lines[0]) +hf, (hb, ha) = plt.subplots(1,2) +hf, hpf = ptt.plot_with_bounds(ha, x_plot, y_plot, rel_sigma, c=ptt.lines[0]) ha.imshow(p_y, origin='lower', extent=[x_plot[0], x_plot[-1], 0.01, 0.99]) if learner.GP.x_train.shape[0]>0: From 5a29593b2244ea12cd02735d009e08a917d29691 Mon Sep 17 00:00:00 2001 From: Nicholas Lawrance Date: Mon, 3 Apr 2017 19:32:27 -0700 Subject: [PATCH 10/38] :poop: :fire: Fixed error in PrefProbit derivatives - Fixed error with indexing the same point multiple times in the relative ratings derivative, Thane look for ^^FIXED BIT - THANE LOOK HERE^^ - Also changed all the references from X_train to X_rel for relative training data for clarity - Moved some test data stuff (sample functions etc) into new file test_data --- GP_preference_demo.py | 131 ++++++++----------------- GPpref.py | 147 ++++++++++++++++++---------- active_learners.py | 209 ++++++++++++++++++++++++++-------------- plot_tools.py | 72 +++++++++++++- pref_active_learning.py | 170 ++++++++++++-------------------- test_data.py | 31 ++++++ 6 files changed, 437 insertions(+), 323 deletions(-) create mode 100644 test_data.py diff --git a/GP_preference_demo.py b/GP_preference_demo.py index 36323a9..fa194bd 100644 --- a/GP_preference_demo.py +++ b/GP_preference_demo.py @@ -4,17 +4,23 @@ import GPpref import scipy.optimize as op import plot_tools as ptt +import test_data # from scipy.stats import beta plt.rc('font',**{'family':'serif','sans-serif':['Computer Modern Roman']}) plt.rc('text', usetex=True) +np.random.seed(1) -train_hyper = True +train_hyper = False +use_test_data = True +verbose = True -log_hyp = np.log([0.2, 0.5, 0.1, 1.0, 10.0]) # length_scale/s, sigma_f, sigma_n_abs, sigma_beta, v_beta -np.random.seed(1) +#log_hyp = np.log([0.2, 0.5, 0.1, 1.0, 10.0]) # length_scale/s, sigma_f, sigma_n_abs, sigma_beta, v_beta +# log_hyp = np.log([0.07, 1.0, 0.25, 1.0, 28.1]) +log_hyp = np.log([0.065, 0.8, 0.8, 0.8, 20.0]) +np.random.seed(0) -n_rel_train = 30 -n_abs_train = 30 +n_rel_train = 3 +n_abs_train = 1 rel_sigma = 0.2 delta_f = 1e-5 @@ -24,15 +30,9 @@ n_xplot = 101 n_mcsamples = 1000 n_ysamples = 101 -marker_options = {'mec':'k', 'mew':0.5} # Define polynomial function to be modelled -def true_function(x): - y = np.cos(6 * np.pi * (x - 0.5)) * np.exp(-10 * (x - 0.5) ** 2) - #y = (np.sin(x*2*np.pi + np.pi/4))/1.2 - #y = np.sin(x*2.0*np.pi + np.pi/4.0) - return y - +true_function = test_data.zero_fun rel_obs_fun = GPpref.RelObservationSampler(true_function, GPpref.PrefProbit(sigma=rel_sigma)) abs_obs_fun = GPpref.AbsObservationSampler(true_function, GPpref.AbsBoundProbit(sigma=beta_sigma, v=beta_v)) @@ -41,33 +41,27 @@ def true_function(x): # True function x_plot = np.linspace(0.0,1.0,n_xplot,dtype='float') x_test = np.atleast_2d(x_plot).T +f_true = abs_obs_fun.f(x_test) +mu_true = abs_obs_fun.mean_link(x_test) +mc_samples = np.random.normal(size=n_mcsamples) +abs_y_samples = np.atleast_2d(np.linspace(0.01, 0.99, n_ysamples)).T +p_abs_y_true = abs_obs_fun.observation_likelihood_array(x_test, abs_y_samples) +p_rel_y_true = rel_obs_fun.observation_likelihood_array(x_test) # Training data - this is a bit weird, but we sample x values, then the uv pairs # are actually indexes into x, because it is easier computationally. You can # recover the actual u,v values using x[ui],x[vi] -if n_rel_train > 0: - x_train = np.random.random((2*n_rel_train,1)) - uvi_train = np.random.choice(range(2*n_rel_train), (n_rel_train,2), replace=False) - uv_train = x_train[uvi_train][:,:,0] - - # Get labels (y), and noisy observations f(uv) and corresponding ranks y_train - y_train, fuv_train = rel_obs_fun.generate_observations(uv_train) - +if use_test_data: + x_rel, uvi_rel, uv_rel, y_rel, fuv_rel, x_abs, y_abs, mu_abs = test_data.data1() else: - x_train = np.zeros((0,1)) - uvi_train = np.zeros((0,2)) - uv_train = np.zeros((0,2)) - y_train = np.zeros((0,1)) - fuv_train = np.zeros((0,2)) - -# For absolute points, get observation y and beta mean mu -x_abs_train = np.random.random((n_abs_train,1)) -y_abs_train, mu_abs_train = abs_obs_fun.generate_observations(x_abs_train) + x_rel, uvi_rel, uv_rel, y_rel, fuv_rel = rel_obs_fun.generate_n_observations(n_rel_train) + x_abs, y_abs, mu_abs = abs_obs_fun.generate_n_observations(n_abs_train) -prefGP = GPpref.PreferenceGaussianProcess(x_train, uvi_train, x_abs_train, y_train, y_abs_train, +# Construct GP object +prefGP = GPpref.PreferenceGaussianProcess(x_rel, uvi_rel, x_abs, y_rel, y_abs, delta_f=delta_f, rel_likelihood=GPpref.PrefProbit(), - abs_likelihood=GPpref.AbsBoundProbit()) + abs_likelihood=GPpref.AbsBoundProbit(), verbose=verbose) # If training hyperparameters, use external optimiser if train_hyper: @@ -82,70 +76,25 @@ def true_function(x): # Expected values E_y = prefGP.abs_posterior_mean(x_test, fhat, vhat) -# Absolute posterior likelihood (MC sampled) -mc_samples = np.random.normal(size=n_mcsamples) -abs_y_samples = np.atleast_2d(np.linspace(0.01, 0.99, n_ysamples)).T -p_y = prefGP.abs_posterior_likelihood(abs_y_samples, fhat=fhat, varhat=vhat, normal_samples=mc_samples) - -# PLOTTING +# Posterior likelihoods (MC sampled for absolute) +p_abs_y_post = prefGP.abs_posterior_likelihood(abs_y_samples, fhat=fhat, varhat=vhat, normal_samples=mc_samples) +p_rel_y_post = prefGP.rel_posterior_likelihood_array(fhat=fhat, varhat=vhat) -# Plot true function, likelihoods and observations -fig_t, (ax_t_l, ax_t_a, ax_t_r) = ptt.plot_setup_2d(t_l=r'True latent function, $f(x)$') -# True latent -f_true = abs_obs_fun.f(x_test) -ptt.plot_with_bounds(ax_t_l, x_test, f_true, rel_sigma, c=ptt.lines[0]) - -# True absolute likelihood -p_abs_y_true = abs_obs_fun.observation_likelihood_array(x_test, abs_y_samples) -abs_extent = [x_test[0,0], x_test[-1,0], abs_y_samples[0,0], abs_y_samples[-1,0]] -h_pat = ax_t_a.imshow(p_abs_y_true, origin='lower', extent=abs_extent) -if x_abs_train.shape[0]>0: - ax_t_a.plot(x_abs_train, y_abs_train, 'w+') -mu_true = abs_obs_fun.mean_link(x_test) -h_yt, = ax_t_a.plot(x_test, mu_true, c=ptt.lines[0]) -ax_t_a.legend([h_yt], ['$E(y|f(x))$']) -fig_t.colorbar(h_pat, ax=ax_t_a) - -# True relative likelihood -p_rel_y_true = rel_obs_fun.observation_likelihood_array(x_test) -rel_y_extent = [x_test.min(), x_test.max(), x_test.min(), x_test.max()] -h_prt = ptt.plot_relative_likelihood(ax_t_r, p_rel_y_true, extent=rel_y_extent) -class_icons = ['ko','wo'] -if x_train.shape[0] > 0: - for uv, fuv, y in zip(uv_train, fuv_train, y_train): - ax_t_r.plot(uv[0], uv[1], class_icons[(y[0]+1)/2], **marker_options) - ax_t_l.plot(uv, fuv, 'b-', color=ptt.lighten(ptt.lines[0])) - ax_t_l.plot(uv[(y+1)/2], fuv[(y+1)/2], class_icons[(y[0]+1)/2], **marker_options) # '+', color=ptt.darken(ptt.lines[0], 1.5) +# Plot true functions +fig_t, (ax_t_l, ax_t_a, ax_t_r) = ptt.true_plots(x_test, f_true, mu_true, rel_sigma, + abs_y_samples, p_abs_y_true, p_rel_y_true, + x_abs, y_abs, uv_rel, fuv_rel, y_rel, + t_l=r'True latent function, $f(x)$') # Posterior estimates -fig_p, (ax_p_l, ax_p_a, ax_p_r) = ptt.plot_setup_2d( - t_a=r'Posterior absolute likelihood, $p(y | \mathcal{Y}, \theta)$', - t_r=r'Posterior relative likelihood $P(x_0 \succ x_1 | \mathcal{Y}, \theta)$') - -# Latent function -hf, hpf = ptt.plot_with_bounds(ax_p_l, x_test, f_true, rel_sigma, c=ptt.lines[0]) - -hf_hat, hpf_hat = ptt.plot_with_bounds(ax_p_l, x_test, fhat, np.sqrt(np.atleast_2d(vhat.diagonal()).T), c=ptt.lines[1]) -ax_p_l.legend([hf, hf_hat], [r'True latent function, $f(x)$', r'$\mathcal{GP}$ estimate $\hat{f}(x)$']) - -# Absolute posterior likelihood -h_pap = ax_p_a.imshow(p_y, origin='lower', extent=abs_extent) -h_yt, = ax_p_a.plot(x_test, mu_true, c=ptt.lines[0]) -hEy, = ax_p_a.plot(x_plot, E_y, color=ptt.lines[3]) -if x_abs_train.shape[0]>0: - ax_p_a.plot(x_abs_train, y_abs_train, 'w+') -ax_p_a.legend([h_yt, hEy], [r'True mean, $E_{p(y|f(x))}[y]$', r'Posterior mean, $E_{p(y|\mathcal{Y})}\left[y\right]$']) -fig_p.colorbar(h_pap, ax=ax_p_a) - -# Relative posterior likelihood -p_rel_y_post = prefGP.rel_posterior_likelihood_array(fhat=fhat, varhat=vhat) -h_prp = ptt.plot_relative_likelihood(ax_p_r, p_rel_y_post, extent=rel_y_extent) -if x_train.shape[0] > 0: - for uv, fuv, y in zip(uv_train, fuv_train, y_train): - ax_p_r.plot(uv[0], uv[1], class_icons[(y[0]+1)/2], **marker_options) - ax_p_l.plot(uv, fuv, 'b-', color=ptt.lighten(ptt.lines[0])) - ax_p_l.plot(uv[(y+1)/2], fuv[(y+1)/2], class_icons[(y[0]+1)/2], **marker_options) # '+', color=ptt.darken(ptt.lines[0], 1.5) +fig_p, (ax_p_l, ax_p_a, ax_p_r) = \ + ptt.estimate_plots(x_test, f_true, mu_true, fhat, vhat, E_y, rel_sigma, + abs_y_samples, p_abs_y_post, p_rel_y_post, + x_abs, y_abs, uv_rel, fuv_rel, y_rel, + t_a=r'Posterior absolute likelihood, $p(y | \mathcal{Y}, \theta)$', + t_r=r'Posterior relative likelihood $P(x_0 \succ x_1 | \mathcal{Y}, \theta)$') + plt.show() diff --git a/GPpref.py b/GPpref.py index 2a8b2aa..daf542c 100644 --- a/GPpref.py +++ b/GPpref.py @@ -17,6 +17,12 @@ def std_norm_cdf(x): #x = np.clip(x, -30, 100 ) return norm.cdf(x) +def norm_pdf_norm_cdf_ratio(z): + # Inverse Mills ratio for stability + out = -z + out[z>-30] = std_norm_pdf(z[z>-30])/std_norm_cdf(z[z>-30]) + return out + # Define squared distance calculation function def squared_distance(A,B): A = np.reshape(A,(len(A),1)) @@ -89,16 +95,20 @@ def I_k(self, x, uv): # Jensen and Nielsen def derivatives(self, uvi, y, f): nx = len(f) z = self.z_k(y=y, f=self.get_rel_f(f, uvi)) - phi_z = std_norm_cdf(z) - N_z = std_norm_pdf(z) + N_over_phi = norm_pdf_norm_cdf_ratio(z) + # phi_z = std_norm_cdf(z) + # N_z = std_norm_pdf(z) # First derivative (Jensen and Nielsen) dpy_df = np.zeros((nx, 1), dtype='float') - dpyuv_df = y * self._isqrt2sig * N_z / phi_z - dpy_df[uvi[:, 0]] += -dpyuv_df # This implements I_k (note switch because Jensen paper has funky backwards z_k) - dpy_df[uvi[:, 1]] += dpyuv_df - - inner = -self._i2var * (z * N_z / phi_z + (N_z / phi_z) ** 2) + dpyuv_df = y * self._isqrt2sig * N_over_phi # N_z / phi_z + for i, (uvii, uvij) in enumerate(uvi): + dpy_df[uvii] += -dpyuv_df[i] ## ^^FIXED BIT - THANE LOOK HERE^^ + dpy_df[uvij] += dpyuv_df[i] # This implements I_k (note switch because Jensen paper has funky backwards z_k) + # dpy_df[uvi[:, 0]] += -dpyuv_df # NOTE: THESE TWO LINES ARE INCORRECT FOR REPEATED INDEXES!!!!! + # dpy_df[uvi[:, 1]] += dpyuv_df # This implements I_k (note switch because Jensen paper has funky backwards z_k) + + inner = -self._i2var * (z * N_over_phi + (N_over_phi) ** 2) W = np.zeros((nx, nx), dtype='float') for uvik, ddpy_df in zip(uvi, inner): xi, yi = uvik @@ -107,7 +117,8 @@ def derivatives(self, uvi, y, f): W[xi, yi] -= -ddpy_df # Otherwise, I(x_i)*I(y_i) = -1*1 = -1 W[yi, xi] -= -ddpy_df - return W, dpy_df + py = np.log(std_norm_cdf(z)) + return W, dpy_df, py def get_rel_f(self, f, uvi): return np.hstack((f[uvi[:, 0]], f[uvi[:, 1]])) @@ -178,18 +189,19 @@ def log_likelihood(self, y, f): def derivatives(self, y, f): - alpha = self.alpha(f) - beta = self.beta(f) #let's make a distribution called beta that also has beta as a parameter! + aa, bb = self.get_alpha_beta(f) - dpy_df = self.v*self._isqrt2sig*std_norm_pdf(f*self._isqrt2sig) * (np.log(y)-np.log(1-y) - digamma(alpha) + digamma(beta) ) + dpy_df = self.v*self._isqrt2sig*std_norm_pdf(f*self._isqrt2sig) * (np.log(y) - np.log(1-y) - digamma(aa) + digamma(bb)) Wdiag = - self.v*self._isqrt2sig*std_norm_pdf(f*self._isqrt2sig) * ( - f*self._i2var*( np.log(y)-np.log(1.0-y)-digamma(alpha) + digamma(beta) ) + - self.v*self._isqrt2sig*std_norm_pdf(f*self._isqrt2sig) * (polygamma(1, alpha) + polygamma(1, beta)) ) + f * self._i2var * (np.log(y) - np.log(1.0-y) - digamma(aa) + digamma(bb)) + + self.v * self._isqrt2sig * std_norm_pdf(f*self._isqrt2sig) * (polygamma(1, aa) + polygamma(1, bb)) ) W = np.diagflat(Wdiag) - return -W, dpy_df + py = np.log(beta.pdf(y, aa, bb)) + + return -W, dpy_df, py def posterior_mean(self, fhat, var_star): #mu_t = self.mean_link(fhat) @@ -207,52 +219,59 @@ def generate_samples(self, f): class PreferenceGaussianProcess(object): - def __init__(self, x_train, uvi_train, x_abs_train, y_train, y_abs_train, rel_likelihood=PrefProbit(), delta_f = 1e-6, abs_likelihood=AbsBoundProbit()): + def __init__(self, x_rel, uvi_rel, x_abs, y_rel, y_abs, rel_likelihood=PrefProbit(), delta_f = 1e-6, + abs_likelihood=AbsBoundProbit(), verbose=False): # log_hyp are log of hyperparameters, note that it is [length_0, ..., length_d, sigma_f, sigma_probit, v_beta] # Training points are split into relative and absolute for calculating f, but combined for predictions. - self.set_observations(x_train, uvi_train, x_abs_train, y_train, y_abs_train) + self.set_observations(x_rel, uvi_rel, x_abs, y_rel, y_abs) self.delta_f = delta_f self.rel_likelihood = rel_likelihood self.abs_likelihood = abs_likelihood + self.verbose = verbose + self.kern = GPy.kern.RBF(self._xdim, ARD=True) self.Ix = np.eye(self._nx) self.f = None + self.init_extras() + + def init_extras(self): + pass - def set_observations(self, x_train, uvi_train, x_abs_train, y_train, y_abs_train): - if x_train.shape[0] is not 0: - self._xdim = x_train.shape[1] - elif x_abs_train.shape[0] is not 0: - self._xdim = x_abs_train.shape[1] + def set_observations(self, x_rel, uvi_rel, x_abs, y_rel, y_abs): + if x_rel.shape[0] is not 0: + self._xdim = x_rel.shape[1] + elif x_abs.shape[0] is not 0: + self._xdim = x_abs.shape[1] else: raise Exception("No Input Points") - self._n_rel = x_train.shape[0] - self._n_abs = x_abs_train.shape[0] - self.x_train = x_train - self.y_train = y_train - self.x_abs_train = x_abs_train - self.y_abs_train = y_abs_train - self.uvi_train = uvi_train + self._n_rel = x_rel.shape[0] + self._n_abs = x_abs.shape[0] + self.x_rel = x_rel + self.y_rel = y_rel + self.x_abs = x_abs + self.y_abs = y_abs + self.uvi_rel = uvi_rel - self.x_train_all = np.concatenate((self.x_train, self.x_abs_train), 0) + self.x_train_all = np.concatenate((self.x_rel, self.x_abs), 0) self._nx = self.x_train_all.shape[0] self.Ix = np.eye(self._nx) def add_observations(self, x, y, uvi=None): if uvi is None: - x_abs_train = np.concatenate((self.x_abs_train, x), 0) - y_abs_train = np.concatenate((self.y_abs_train, y), 0) - self.set_observations(self.x_train, self.uvi_train, x_abs_train, self.y_train, y_abs_train) + x_abs = np.concatenate((self.x_abs, x), 0) + y_abs = np.concatenate((self.y_abs, y), 0) + self.set_observations(self.x_rel, self.uvi_rel, x_abs, self.y_rel, y_abs) else: - x_train = np.concatenate((self.x_train, x), 0) - y_train = np.concatenate((self.y_train, y), 0) - uvi_train = np.concatenate((self.uvi_train, uvi+self.x_train.shape[0]), 0) - self.set_observations(x_train, uvi_train, self.x_abs_train, y_train, self.y_abs_train) + x_rel = np.concatenate((self.x_rel, x), 0) + y_rel = np.concatenate((self.y_rel, y), 0) + uvi_rel = np.concatenate((self.uvi_rel, uvi + self.x_rel.shape[0]), 0) + self.set_observations(x_rel, uvi_rel, self.x_abs, y_rel, self.y_abs) def calc_laplace(self, loghyp): self.kern.lengthscale = np.exp(loghyp[0:self._xdim]) @@ -273,6 +292,7 @@ def calc_laplace(self, loghyp): # First, solve for \hat{f} and W (mode finding Laplace approximation, Newton-Raphson) f_error = self.delta_f + 1 + nloops = 0 while f_error > self.delta_f: # Is splitting these apart correct? Will there be off-diagonal elements of W_abs that should be @@ -282,20 +302,21 @@ def calc_laplace(self, loghyp): # Get relative Hessian and Gradient if self._n_rel>0 and self._n_abs>0: - W_rel, dpy_df_rel = self.rel_likelihood.derivatives(self.uvi_train, self.y_train, f_rel) + W_rel, dpy_df_rel, py_rel = self.rel_likelihood.derivatives(self.uvi_rel, self.y_rel, f_rel) # Get Absolute Hessian and Gradient - # Note that y_abs_train has to be [0,1], which could be an issue. - W_abs, dpy_df_abs = self.abs_likelihood.derivatives(self.y_abs_train, f_abs) + # Note that y_abs has to be in [0,1] + W_abs, dpy_df_abs, py_abs = self.abs_likelihood.derivatives(self.y_abs, f_abs) # Combine W, gradient + py = py_abs.sum() + py_rel.sum() dpy_df = np.concatenate((dpy_df_rel, dpy_df_abs), axis=0) W = block_diag(W_rel, W_abs) elif self._n_rel>0: - W, dpy_df = self.rel_likelihood.derivatives(self.uvi_train, self.y_train, f_rel) + W, dpy_df, py = self.rel_likelihood.derivatives(self.uvi_rel, self.y_rel, f_rel) elif self._n_abs>0: - W, dpy_df = self.abs_likelihood.derivatives(self.y_abs_train, f_abs) + W, dpy_df, py = self.abs_likelihood.derivatives(self.y_abs, f_abs) # # print "Total" # print "Dpy, W:" @@ -305,7 +326,7 @@ def calc_laplace(self, loghyp): g = (self.iK + W - lambda_eye) f_new = np.matmul(np.linalg.inv(g), np.matmul(W-lambda_eye, f) + dpy_df) - #lml = self.rel_likelihood.log_marginal(self.uvi_train, self.y_train, f_new, iK, logdetK) + #lml = self.rel_likelihood.log_marginal(self.uvi_rel, self.y_rel, f_new, iK, logdetK) ## Jensen version (iK + W)^-1 = K - K((I + WK)^-1)WK (not sure how to get f'K^-1f though... # ig = K - np.matmul(np.matmul(np.matmul(K, np.linalg.inv(Ix + np.matmul(W, K))), W), K) @@ -318,6 +339,12 @@ def calc_laplace(self, loghyp): # print "F Error: " + str(f_error) #,lml # print "F New: " + str(f_new) f = f_new + nloops += 1 + if nloops > 10000: + raise RuntimeError("Maximum loops exceeded in calc_laplace!!") + if self.verbose: + lml = py - 0.5*np.matmul(f.T, np.matmul(self.iK, f)) - 0.5*np.log(np.linalg.det(np.matmul(W, self.Kxx) + self.Ix)) + print "Laplace iteration {0:02d}, log p(y|f) = {1:0.2f}".format(nloops, lml[0,0]) self.W = W self.f = f @@ -327,7 +354,7 @@ def calc_laplace(self, loghyp): return f#, lml def _safe_invert_noise(self, mat): - eps = 1e-6 + eps = 1.0e-6 inv_ok = False while not inv_ok: @@ -338,17 +365,16 @@ def _safe_invert_noise(self, mat): logdet = np.sum(np.log(L.diagonal())) inv_ok = True except np.linalg.linalg.LinAlgError: - eps = eps*10 + eps = max(1e-6, eps*10.0) print "Inversion issue, adding noise: {0}".format(eps) return imat, logdet - def calc_nlml(self, loghyp): - f = self.calc_laplace(loghyp) - if self.f is None: - self.f = f + def calc_nlml(self, loghyp, f=None): + if f is None: + f = self.calc_laplace(loghyp) # Now calculate the log likelihoods (remember log(ax) = log a + log x) - log_py_f_rel = self.rel_likelihood.log_likelihood(self.y_train, self.rel_likelihood.get_rel_f(f, self.uvi_train)) - log_py_f_abs = self.abs_likelihood.log_likelihood(self.y_abs_train, f[self._n_rel:]) #TODO: I would prefer to use the indexing in absolute ratings too for consistency + log_py_f_rel = self.rel_likelihood.log_likelihood(self.y_rel, self.rel_likelihood.get_rel_f(f, self.uvi_rel)) + log_py_f_abs = self.abs_likelihood.log_likelihood(self.y_abs, f[self._n_rel:]) #TODO: I would prefer to use the indexing in absolute ratings too for consistency fiKf = np.matmul(f.T, self.iKf) lml = log_py_f_rel.sum()+log_py_f_abs.sum() - 0.5*fiKf - 0.5*np.log(np.linalg.det(self.KWI)) return -lml @@ -358,6 +384,7 @@ def predict_latent(self, x): kt = self.kern.K(self.x_train_all, x) mean_latent = np.matmul(kt.T, self.iKf) Ktt = self.kern.K(x) + # iKW, _ = self._safe_invert_noise(self.KWI) iKW = np.linalg.inv(self.KWI) var_latent = Ktt - np.matmul(kt.T, np.matmul(iKW, np.matmul(self.W, kt))) return mean_latent, var_latent @@ -427,6 +454,14 @@ def generate_observations(self, x): y, ff = self.l.generate_samples(fx) return y, ff + @staticmethod + def _gen_x_obs(n, n_xdim=1, domain=None): + # Domain should be 2 x n_xdim, i.e [[x0_lo, x1_lo, ... , xn_lo], [x0_hi, x1_hi, ... , xn_hi ]] + x_test = np.random.uniform(size=(n, n_xdim)) + if domain is not None: + x_test = x_test * np.diff(domain, axis=0) + domain[0, :] + return x_test + class AbsObservationSampler(ObservationSampler): def observation_likelihood_array(self, x, y): @@ -440,6 +475,11 @@ def mean_link(self, x): fx = self.f(x) return self.l.mean_link(fx) + def generate_n_observations(self, n, n_xdim=1, domain=None): + x = self._gen_x_obs(n, n_xdim, domain) + y, mu = self.generate_observations(x) + return x, y, mu + class RelObservationSampler(ObservationSampler): def observation_likelihood_array(self, x, y=-1): @@ -451,7 +491,12 @@ def observation_likelihood_array(self, x, y=-1): p_y[:, i:i + 1] = self.l.likelihood(y, cross_fx) return p_y - + def generate_n_observations(self, n, n_xdim=1, domain=None): + x = self._gen_x_obs(2*n, n_xdim, domain) + uvi = np.arange(2*n).reshape((n, 2)) + uv = x[uvi][:, :, 0] + y, fuv = self.generate_observations(uv) + return x, uvi, uv, y, fuv diff --git a/active_learners.py b/active_learners.py index bbfa9c9..fcc6880 100644 --- a/active_learners.py +++ b/active_learners.py @@ -1,112 +1,179 @@ import numpy as np import GPpref from scipy.stats import beta +import plot_tools as ptt -class ActiveLearner(object): - def __init__(self, x_train, uvi_train, x_abs_train, y_train, y_abs_train, delta_f, abs_likelihood): +def calc_ucb(fhat, vhat, gamma=2.0): + return fhat + gamma * np.sqrt(np.atleast_2d(vhat.diagonal()).T) - self.nx_dim = x_abs_train.shape[1] - self.GP = GPpref.PreferenceGaussianProcess(x_train, uvi_train, x_abs_train, y_train, y_abs_train, - delta_f=delta_f, - abs_likelihood=abs_likelihood) - self.add_observations = self.GP.add_observations +class ActiveLearner(GPpref.PreferenceGaussianProcess): + def init_extras(self): + self._default_uvi = np.array([[0, 1]]) + self._plus_y_obs = np.ones((1, 1), dtype='int') + self._minus_y_obs = -1*self._plus_y_obs def set_hyperparameters(self, log_hyp): self.log_hyp = log_hyp - def calc_laplace(self): - self.f = self.GP.calc_laplace(self.log_hyp) + def solve_laplace(self, log_hyp=None): + if log_hyp is None: + log_hyp = self.log_hyp + self.f = self.calc_laplace(log_hyp) return self.f - def predict_latent(self, x_test): - fhat, vhat = self.GP.predict_latent(x_test) - vhat = np.atleast_2d(vhat.diagonal()).T - return fhat, vhat - - def expected_y(self, x_test, fhat, vhat): - # Expected values0 - return self.GP.expected_y(x_test, fhat, vhat) - - def abs_posterior_likelihood(self, fhat, vhat, y_samples, mc_samples): - # Sampling from posterior to generate output pdfs - p_y = np.zeros((len(y_samples), len(fhat))) - iny = 1.0/len(y_samples) - for i,(fstar,vstar) in enumerate(zip(fhat, vhat)): - f_samples = mc_samples*vstar+fstar - aa, bb = self.GP.abs_likelihood.get_alpha_beta(f_samples) - p_y[:, i] = [iny*np.sum(beta.pdf(yj, aa, bb)) for yj in y_samples] - return p_y - def get_observations(self): - return self.GP.x_train, self.GP.uvi_train, self.GP.x_abs_train, self.GP.y_train, self.GP.y_abs_train + return self.x_rel, self.uvi_rel, self.x_abs, self.y_rel, self.y_abs - def select_observation(self, domain=[0.0, 1.0]): + def select_observation(self, domain=None): if np.random.uniform() < 0.5: - return np.random.uniform(low=domain[0], high=domain[1], size=(1,1)) + return self.uniform_domain_sampler(1, domain), None else: - return np.random.uniform(low=domain[0], high=domain[1], size=(2,1)) + return self.uniform_domain_sampler(2, domain), np.array([[0, 1]]) + + def uniform_domain_sampler(self, n_samples, domain=None): + # Domain should be 2 x n_xdim, i.e [[x0_lo, x1_lo, ... , xn_lo], [x0_hi, x1_hi, ... , xn_hi ]] + x_test = np.random.uniform(size=(n_samples, self._xdim)) + if domain is not None: + x_test = x_test*np.diff(domain, axis=0) + domain[0, :] + return x_test - #def init_plots(self): - + def create_posterior_plot(self, x_test, f_true, mu_true, rel_sigma, fuv_train, abs_y_samples, mc_samples): + # Latent predictions + fhat, vhat = self.predict_latent(x_test) + # Expected values + E_y = self.abs_posterior_mean(x_test, fhat, vhat) + + # Absolute posterior likelihood (MC sampled) + # Posterior likelihoods (MC sampled for absolute) + p_abs_y_post = self.abs_posterior_likelihood(abs_y_samples, fhat=fhat, varhat=vhat, normal_samples=mc_samples) + p_rel_y_post = self.rel_posterior_likelihood_array(fhat=fhat, varhat=vhat) + x_train, uvi_train, x_abs_train, y_train, y_abs_train = self.get_observations() + uv_train = x_train[uvi_train][:, :, 0] + + # Posterior estimates + fig_p, (ax_p_l, ax_p_a, ax_p_r) = \ + ptt.estimate_plots(x_test, f_true, mu_true, fhat, vhat, E_y, rel_sigma, + abs_y_samples, p_abs_y_post, p_rel_y_post, + x_abs_train, y_abs_train, uv_train, fuv_train, y_train, + t_a=r'Posterior absolute likelihood, $p(y | \mathcal{Y}, \theta)$', + t_r=r'Posterior relative likelihood $P(x_0 \succ x_1 | \mathcal{Y}, \theta)$') + return fig_p, (ax_p_l, ax_p_a, ax_p_r) class UCBLatent(ActiveLearner): - def select_observation(self, domain=[0.0, 1.0], ntest=100, gamma=2.0): - x_test = np.random.uniform(low=domain[0], high=domain[1], size=(ntest, 1)) + # All absolute returns + def select_observation(self, domain=None, ntest=100, gamma=2.0): + x_test = self.uniform_domain_sampler(ntest, domain) fhat, vhat = self.predict_latent(x_test) - return x_test[np.argmax(fhat + gamma*np.sqrt(vhat))] + ucb = calc_ucb(fhat, vhat, gamma) + return x_test[np.argmax(ucb), :], None class UCBOut(ActiveLearner): - def select_observation(self, domain=[0.0, 1.0], ntest=100, gamma=2.0): + def select_observation(self, domain=None, ntest=100, gamma=2.0): # Don't know how to recover the second moment of the predictive distribution, so this isn't done - x_test = np.random.uniform(low=domain[0], high=domain[1], size=(ntest, 1)) + x_test = self.uniform_domain_sampler(ntest, domain) fhat, vhat = self.predict_latent(x_test) Ey = self.expected_y(x_test, fhat, vhat) - return x_test[np.argmax(Ey)] + return x_test[np.argmax(Ey), :], None class PeakComparitor(ActiveLearner): - def test_observation(self, x, y, uv, x_test, gamma): - self.add_observations(x, y, uv) - f = self.calc_laplace() + def test_observation(self, x, y, x_test, gamma): + self.store_observations() + self.add_observations(x, y, self._default_uvi) + f = self.solve_laplace() fhat, vhat = self.predict_latent(x_test) - max_ucb = (fhat + gamma*np.sqrt(np.atleast_2d(vhat.diagonal()).T)).max() - return max_ucb + ucb = calc_ucb(fhat, vhat, gamma) + self.reset_observations() + return ucb.max() + def store_observations(self): + self.crx, self.cuv, self.cax, self.cry, self.cay = self.get_observations() - def select_observation(self, domain=[0.0, 1.0], ntest=50, gamma=2.0, n_comparitors=1): - crx, cuv, cax, cry, cay = self.get_observations() + def reset_observations(self): + try: + self.set_observations(self.crx, self.cuv, self.cax, self.cry, self.cay) + except AttributeError: + print "reset_observations failed: existing observations not found" - x_test = np.random.uniform(low=domain[0], high=domain[1], size=(ntest, 1)) - fhat, vhat = self.GP.predict_latent(x_test) - max_x = np.argmax(fhat) - other_x = np.delete(np.arange(ntest), max_x) - uv = np.vstack((max_x*np.ones(ntest-1, dtype='int'), other_x)).T + def select_observation(self, domain=None, n_test=50, gamma=2.0, n_comparators=1): + x_test = self.uniform_domain_sampler(n_test, domain) + fhat, vhat = self.predict_latent(x_test) + ucb = calc_ucb(fhat, vhat, gamma) + max_xi = np.argmax(ucb) # Old method used highest x, not ucb + other_xi = np.delete(np.arange(n_test), max_xi) + uvi = np.vstack((max_xi * np.ones(n_test - 1, dtype='int'), other_xi)).T - p_pref = self.GP.rel_likelihood.posterior_likelihood(fhat, vhat, uv, y=-1) - V = np.zeros(ntest-1) - x = np.zeros((2,1), dtype='float') - x[0] = x_test[max_x] - ypos = np.ones((1, 1),dtype='int') - ruv = np.array([[0, 1]]) + p_pref = self.rel_likelihood.posterior_likelihood(fhat, vhat, uvi, y=-1) + V = np.zeros(n_test - 1) + x = np.zeros((2, 1), dtype='float') + x[0] = x_test[max_xi] # Now calculate the expected value for each observation pair - for i,uv1 in enumerate(other_x): - x[1] = x_test[uv1] - V[i] += p_pref[i]*self.test_observation(x, -1*ypos, ruv, x_test, gamma) - self.GP.set_observations(crx, cuv, cax, cry, cay) - V[i] += (1-p_pref[i])*self.test_observation(x, ypos, ruv, x_test, gamma) - self.GP.set_observations(crx, cuv, cax, cry, cay) + for i,uvi1 in enumerate(other_xi): + x[1] = x_test[uvi1] + V[i] += p_pref[i]*self.test_observation(x, self._minus_y_obs, x_test, gamma) + V[i] += (1-p_pref[i])*self.test_observation(x, self._plus_y_obs, x_test, gamma) + + best_n = np.argpartition(V, -n_comparators)[-n_comparators:] + # best = np.argmax(V) + cVmax = np.argmax(ucb) # This is repeated in case I want to change max_xi + + if ucb[cVmax] > V.max(): + return x_test[[cVmax], :], None + else: + xi = np.zeros(n_comparators+1, dtype='int') + xi[0] = max_xi + xi[1:] = best_n + uvi_out = np.zeros(shape=(n_comparators, 2), dtype='int') + uvi_out[:, 1] = np.arange(start=1, stop=n_comparators+1, dtype='int') + return x_test[xi,:], uvi_out - best = np.argmax(V) - x[1] = x_test[uv[best,1]] - cV = gamma*np.sqrt(np.atleast_2d(vhat.diagonal())).T + fhat - cVmax = np.argmax(cV) - if cV[cVmax] > V.max(): - x = np.array([x_test[cVmax]]) - return x +class LikelihoodImprovement(PeakComparitor): + def test_observation(self, x, y, x_test, max_xi): + self.store_observations() + self.add_observations(x, y, self._default_uvi) + f = self.solve_laplace() + fhat, vhat = self.predict_latent(x_test) + new_xi = np.argmax(fhat) + p_new_is_better = self.rel_likelihood.posterior_likelihood(fhat, vhat, np.array([[max_xi, new_xi]]), self._plus_y_obs) + self.reset_observations() + return p_new_is_better + + def select_observation(self, domain=None, n_test=50, req_improvement=0.6, n_comparators=1, gamma=1.5): + x_test = self.uniform_domain_sampler(n_test, domain) + fhat, vhat = self.predict_latent(x_test) + max_xi = np.argmax(fhat) + other_xi = np.delete(np.arange(n_test), max_xi) + uvi = np.vstack((max_xi * np.ones(n_test - 1, dtype='int'), other_xi)).T + + p_pref = self.rel_likelihood.posterior_likelihood(fhat, vhat, uvi, y=-1) + V = np.zeros(n_test - 1) + x = np.zeros((2, 1), dtype='float') + x[0] = x_test[max_xi] + # Now calculate the expected value for each observation pair + for i,uvi1 in enumerate(other_xi): + x[1] = x_test[uvi1] + V[i] += p_pref[i]*self.test_observation(x, self._minus_y_obs, x_test, max_xi) + V[i] += (1-p_pref[i])*self.test_observation(x, self._plus_y_obs, x_test, max_xi) + + best_n = np.argpartition(V, -n_comparators)[-n_comparators:] + # best = np.argmax(V) + print 'V_max = {0}'.format(V.max()) + + if V.max() < req_improvement: + ucb = calc_ucb(fhat, vhat, gamma) + return x_test[[np.argmax(ucb)], :], None + else: + xi = np.zeros(n_comparators+1, dtype='int') + xi[0] = max_xi + xi[1:] = best_n + uvi_out = np.zeros(shape=(n_comparators, 2), dtype='int') + uvi_out[:, 1] = np.arange(start=1, stop=n_comparators+1, dtype='int') + return x_test[xi,:], uvi_out + diff --git a/plot_tools.py b/plot_tools.py index 785b342..d02f9f4 100644 --- a/plot_tools.py +++ b/plot_tools.py @@ -1,3 +1,4 @@ +import os import numpy as np import matplotlib.pyplot as plt from matplotlib.patches import Polygon @@ -50,4 +51,73 @@ def plot_relative_likelihood(ax, p_y, extent): h_pc = ax.contour(p_y, levels=[0.5], origin='lower', linewidths=2, extent=extent) plt.clabel(h_pc, inline=1, fontsize=10) ax.get_figure().colorbar(h_p, ax=ax) - return h_p \ No newline at end of file + return h_p + + +def true_plots(xt, ft, mu_t, rel_sigma, y_samples, p_a_y, p_r_y, xa_train, ya_train, uvr_train, fuvr_train, yr_train, + class_icons=['ko', 'wo'], marker_options={'mec':'k', 'mew':0.5}, *args, **kwargs): + + + # Plot true function, likelihoods and observations + fig, (ax_l, ax_a, ax_r) = plot_setup_2d(**kwargs) + + # True latent + plot_with_bounds(ax_l, xt, ft, rel_sigma, c=lines[0]) + + # True absolute likelihood + abs_extent = [xt[0, 0], xt[-1, 0], y_samples[0, 0], y_samples[-1, 0]] + h_pat = ax_a.imshow(p_a_y, origin='lower', extent=abs_extent) + if xa_train.shape[0] > 0: + ax_a.plot(xa_train, ya_train, 'w+') + h_yt, = ax_a.plot(xt, mu_t, c=lines[0]) + ax_a.legend([h_yt], ['$E(y|f(x))$']) + fig.colorbar(h_pat, ax=ax_a) + + # True relative likelihood + rel_y_extent = [xt[0, 0], xt[-1, 0], xt[0, 0], xt[-1, 0]] + h_prt = plot_relative_likelihood(ax_r, p_r_y, extent=rel_y_extent) + if xt.shape[0] > 0: + for uv, fuv, y in zip(uvr_train, fuvr_train, yr_train): + ax_r.plot(uv[0], uv[1], class_icons[(y[0] + 1) / 2], **marker_options) + ax_l.plot(uv, fuv, 'b-', color=lighten(lines[0])) + ax_l.plot(uv[(y + 1) / 2], fuv[(y + 1) / 2], class_icons[(y[0] + 1) / 2], **marker_options) + return fig, (ax_l, ax_a, ax_r) + + +def estimate_plots(xt, ft, mu_t, fhat, vhat, E_y, rel_sigma, + y_samples, p_a_y, p_r_y, xa_train, ya_train, uvr_train, fuvr_train, yr_train, + class_icons = ['ko', 'wo'], marker_options = {'mec':'k', 'mew':0.5}, *args, **kwargs): + fig, (ax_l, ax_a, ax_r) = plot_setup_2d(**kwargs) + + # Latent function + hf, hpf = plot_with_bounds(ax_l, xt, ft, rel_sigma, c=lines[0]) + + hf_hat, hpf_hat = plot_with_bounds(ax_l, xt, fhat, np.sqrt(np.atleast_2d(vhat.diagonal()).T), c=lines[1]) + ax_l.legend([hf, hf_hat], [r'True latent function, $f(x)$', r'$\mathcal{GP}$ estimate $\hat{f}(x)$']) + + # Absolute posterior likelihood + abs_extent = [xt[0, 0], xt[-1, 0], y_samples[0, 0], y_samples[-1, 0]] + h_pap = ax_a.imshow(p_a_y, origin='lower', extent=abs_extent) + h_yt, = ax_a.plot(xt, mu_t, c=lines[0]) + hEy, = ax_a.plot(xt, E_y, color=lines[3]) + if xa_train.shape[0] > 0: + ax_a.plot(xa_train, ya_train, 'w+') + ax_a.legend([h_yt, hEy], + [r'True mean, $E_{p(y|f(x))}[y]$', r'Posterior mean, $E_{p(y|\mathcal{Y})}\left[y\right]$']) + fig.colorbar(h_pap, ax=ax_a) + + # Relative posterior likelihood + rel_y_extent = [xt[0, 0], xt[-1, 0], xt[0, 0], xt[-1, 0]] + h_prp = plot_relative_likelihood(ax_r, p_r_y, extent=rel_y_extent) + if uvr_train.shape[0] > 0: + for uv, fuv, y in zip(uvr_train, fuvr_train, yr_train): + ax_r.plot(uv[0], uv[1], class_icons[(y[0] + 1) / 2], **marker_options) + ax_l.plot(uv, fuv, 'b-', color=lighten(lines[0])) + ax_l.plot(uv[(y + 1) / 2], fuv[(y + 1) / 2], class_icons[(y[0] + 1) / 2], **marker_options) + + return fig, (ax_l, ax_a, ax_r) + +def ensure_dir(file_path): + directory = os.path.dirname(file_path) + if not os.path.exists(directory): + os.makedirs(directory) \ No newline at end of file diff --git a/pref_active_learning.py b/pref_active_learning.py index 03937d0..c2b3d94 100644 --- a/pref_active_learning.py +++ b/pref_active_learning.py @@ -1,22 +1,25 @@ # Simple 1D GP classification example +import time import numpy as np import matplotlib.pyplot as plt import GPpref -import scipy.optimize as op import plot_tools as ptt -import mcmc -from scipy.stats import beta import active_learners +import test_data + +nowstr = time.strftime("%Y_%m_%d-%H_%M") plt.rc('font',**{'family':'serif','sans-serif':['Computer Modern Roman']}) plt.rc('text', usetex=True) +save_plots = True + # log_hyp = np.log([0.1,0.5,0.1,10.0]) # length_scale, sigma_f, sigma_probit, v_beta -log_hyp = np.log([0.07, 0.6, 0.25, 1.0, 28.1]) -np.random.seed(3) +log_hyp = np.log([0.07, 0.75, 0.25, 1.0, 28.1]) +np.random.seed(0) -n_rel_train = 2 +n_rel_train = 1 n_abs_train = 1 -rel_sigma = 0.05 +rel_sigma = 0.2 delta_f = 1e-5 beta_sigma = 0.8 @@ -28,126 +31,75 @@ n_queries = 20 -# Define function to be modelled -def true_function(x): - #y = (np.sin(x*2*np.pi + np.pi/4) + 1.25)/2.5 - #y = np.sin(x*2.0*np.pi + np.pi/4.0) - y = np.cos(6 * np.pi * (x - 0.5)) * np.exp(-10 * (x - 0.5) ** 2) - return y +# Define polynomial function to be modelled +true_function = test_data.multi_peak + +if save_plots: + nowstr = time.strftime("%Y_%m_%d-%H_%M") + fig_dir = 'fig/' + nowstr + '/' + ptt.ensure_dir(fig_dir) + print "Figures will be saved to: {0}".format(fig_dir) rel_obs_fun = GPpref.RelObservationSampler(true_function, GPpref.PrefProbit(sigma=rel_sigma)) abs_obs_fun = GPpref.AbsObservationSampler(true_function, GPpref.AbsBoundProbit(sigma=beta_sigma, v=beta_v)) -# Gaussian noise observation function -def normal_obs_function(x): - fx = true_function(x) - noise = np.random.normal(scale=rel_sigma, size=x.shape) - return fx + noise - -beta_obs = GPpref.AbsBoundProbit(sigma=beta_sigma, v=beta_v) - -def beta_obs_function(x): - fx = true_function(x) - mu = GPpref.mean_link(f) - a, b = beta_obs.get_alpha_beta(mu) - z = [beta.rvs(aa, bb) for aa,bb in zip(a,b)] - return z - -def noisy_preference_rank(uv): - fuv = normal_obs_function(uv) - y = -1*np.ones((fuv.shape[0],1),dtype='int') - y[fuv[:,1] > fuv[:,0]] = 1 - return y, fuv - -# Main program # True function x_plot = np.linspace(0.0,1.0,n_xplot,dtype='float') -y_plot = true_function(x_plot) x_test = np.atleast_2d(x_plot).T +f_true = abs_obs_fun.f(x_test) +mu_true = abs_obs_fun.mean_link(x_test) +mc_samples = np.random.normal(size=n_mcsamples) +abs_y_samples = np.atleast_2d(np.linspace(0.01, 0.99, n_ysamples)).T +p_abs_y_true = abs_obs_fun.observation_likelihood_array(x_test, abs_y_samples) +p_rel_y_true = rel_obs_fun.observation_likelihood_array(x_test) -# Training data - this is a bit weird, but we sample x values, then the uv pairs -# are actually indexes into x, because it is easier computationally. You can -# recover the actual u,v values using x[ui],x[vi] -if n_rel_train > 0: - x_train = np.random.random((2*n_rel_train,1)) - uvi_train = np.random.choice(range(2*n_rel_train), (n_rel_train,2), replace=False) - uv_train = x_train[uvi_train][:,:,0] - - # Get noisy observations f(uv) and corresponding ranks y_train - y_train, fuv_train = noisy_preference_rank(uv_train) - -else: - x_train = np.zeros((0,1)) - uvi_train = np.zeros((0,2)) - uv_train = np.zeros((0,2)) - y_train = np.zeros((0,1)) - fuv_train = np.zeros((0,2)) - -x_abs_train = np.random.random((n_abs_train,1)) -y_abs_train = beta_obs_function(x_abs_train, sigma=rel_sigma) -#y_abs_train = np.clip(normal_obs_function(x_abs_train), 0.01, .99) +# Training data +x_rel, uvi_rel, uv_rel, y_rel, fuv_rel = rel_obs_fun.generate_n_observations(n_rel_train, n_xdim=1) +x_abs, y_abs, mu_abs = abs_obs_fun.generate_n_observations(n_abs_train, n_xdim=1) -learner = active_learners.PeakComparitor(x_train, uvi_train, x_abs_train, y_train, y_abs_train, delta_f=delta_f, - abs_likelihood=GPpref.AbsBoundProbit(sigma=beta_sigma, v=beta_v)) +# Plot true functions +fig_t, (ax_t_l, ax_t_a, ax_t_r) = ptt.true_plots(x_test, f_true, mu_true, rel_sigma, + abs_y_samples, p_abs_y_true, p_rel_y_true, + x_abs, y_abs, uv_rel, fuv_rel, y_rel, + t_l=r'True latent function, $f(x)$') +if save_plots: + fig_t.savefig(fig_dir+'true.pdf', bbox_inches='tight') -theta0 = log_hyp +# Construct active learner object +learner = active_learners.LikelihoodImprovement(x_rel, uvi_rel, x_abs, y_rel, y_abs, delta_f=delta_f, + rel_likelihood=GPpref.PrefProbit(), abs_likelihood=GPpref.AbsBoundProbit()) # Get initial solution learner.set_hyperparameters(log_hyp) -f = learner.calc_laplace() +f = learner.solve_laplace() + +if save_plots: + fig_p, (ax_p_l, ax_p_a, ax_p_r) = learner.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, + abs_y_samples, mc_samples) + fig_p.savefig(fig_dir+'posterior00.pdf', bbox_inches='tight') for obs_num in range(n_queries): - fuv = np.array([[0, 1]]) - next_x = np.atleast_2d(learner.select_observation()) - if next_x.shape[0] == 1: - next_y = beta_obs_function(next_x, sigma=rel_sigma) + next_x, next_uvi = learner.select_observation(req_improvement=0.55, gamma=2.0, n_comparators=4) + if next_uvi is None: + next_y, next_f = abs_obs_fun.generate_observations(next_x) learner.add_observations(next_x, next_y) else: - next_y, next_f = noisy_preference_rank(next_x.T) - fuv_train = np.concatenate((fuv_train, next_f), 0) - learner.add_observations(next_x, next_y, fuv) + next_y, next_f = rel_obs_fun.generate_observations(next_x[next_uvi][:,:,0]) + fuv_rel = np.concatenate((fuv_rel, next_f), 0) + learner.add_observations(next_x, next_y, next_uvi) print next_x, next_y - f = learner.calc_laplace() - -# Latent predictions -fhat, vhat = learner.predict_latent(x_test) - -# Expected values -E_y = learner.GP.abs_posterior_mean(x_test, fhat, vhat) - -# Sampling from posterior to show likelihoods -mc_samples = np.random.normal(size=n_mcsamples) -y_samples = np.linspace(0.01, 0.99, n_ysamples) -p_y = learner.GP.abs_posterior_likelihood(y_samples, fhat=fhat, varhat=vhat, normal_samples=mc_samples) - -hf_input, ha_input =plt.subplot(1,1) - - -hf, (hb, ha) = plt.subplots(1,2) -hf, hpf = ptt.plot_with_bounds(ha, x_plot, y_plot, rel_sigma, c=ptt.lines[0]) -ha.imshow(p_y, origin='lower', extent=[x_plot[0], x_plot[-1], 0.01, 0.99]) - -if learner.GP.x_train.shape[0]>0: - for uv,fuv,y in zip(learner.GP.x_train[learner.GP.uvi_train][:,:,0], fuv_train, learner.GP.y_train): - ha.plot(uv, fuv, 'b-', color=ptt.lighten(ptt.lines[0])) - ha.plot(uv[(y+1)/2],fuv[(y+1)/2],'+', color=ptt.darken(ptt.lines[0], 1.5)) - -if learner.GP.x_abs_train.shape[0]>0: - ha.plot(learner.GP.x_abs_train, learner.GP.y_abs_train, 'k+') - -hfhat, hpfhat = ptt.plot_with_bounds(hb, x_test, fhat, np.sqrt(vhat), c=ptt.lines[1]) -hEy, = ha.plot(x_plot, E_y, color=ptt.lines[3]) - -# ha.plot(x_plot, E_y2, color=ptt.lines[4]) -hmap, = ha.plot(x_plot, y_samples[np.argmax(p_y, axis=0)], color='w') -ha.set_title('Training data') -ha.set_ylabel('$y$') -ha.set_xlabel('$x$') -hb.set_xlabel('$x$') -hb.set_ylabel('$f(x)$') - -ha.legend([hf, hEy, hmap], [r'$f(x)$', r'$E_{p(y|\mathcal{Y})}\left[y\right]$', r'$y_{MAP} | \mathcal{Y}$']) -hb.legend([hfhat], [r'Latent function $\hat{f}(x)$']) + f = learner.solve_laplace() + if save_plots: + fig_p, (ax_p_l, ax_p_a, ax_p_r) = learner.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, + abs_y_samples, mc_samples) + fig_p.savefig(fig_dir+'posterior{0:02d}.pdf'.format(obs_num+1), bbox_inches='tight') + plt.close(fig_p) + +learner.print_hyperparameters() + +if not save_plots: + fig_p, (ax_p_l, ax_p_a, ax_p_r) = learner.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, + abs_y_samples, mc_samples) plt.show() diff --git a/test_data.py b/test_data.py new file mode 100644 index 0000000..5116973 --- /dev/null +++ b/test_data.py @@ -0,0 +1,31 @@ +import numpy as np + + +def damped_wave(x): + y = np.cos(6 * np.pi * (x - 0.5)) * np.exp(-10 * (x - 0.5) ** 2) + return y + +def multi_peak(x): + y = np.cos(6 * np.pi * (x - 0.5)) + return y + +def basic_sine(x): + y = (np.sin(x*2*np.pi + np.pi/4))/1.2 + return y + + +def zero_fun(x): + return 0*x + +def data1(): + x_rel = np.array([[0.6], [0.7]]) + uvi_rel = np.array([[0, 1], [1, 0]], dtype='int') + uv_rel = x_rel[uvi_rel][:,:,0] + y_rel = np.array([[1], [1]], dtype='int') + fuv_rel = np.array([[-0.1, 0.1], [-0.1, 0.1]]) + + x_abs = np.array([[0.2]]) + y_abs = np.array([[0.5]]) + mu_abs = np.array([[0.0]]) + + return x_rel, uvi_rel, uv_rel, y_rel, fuv_rel, x_abs, y_abs, mu_abs From 495ca6aa3ec9655ffd1408d3de66e694de2a4826 Mon Sep 17 00:00:00 2001 From: Nicholas Lawrance Date: Thu, 6 Apr 2017 09:24:54 -0700 Subject: [PATCH 11/38] :runner: Added statruns, lots of changes to active learners. --- GP_preference_demo.py | 22 ++++-- GPpref.py | 21 +++-- active_learners.py | 107 +++++++++++++++++-------- active_statruns.py | 168 ++++++++++++++++++++++++++++++++++++++++ plot_statruns.py | 50 ++++++++++++ pref_active_learning.py | 32 +++++--- test_data.py | 35 +++++++++ 7 files changed, 378 insertions(+), 57 deletions(-) create mode 100644 active_statruns.py create mode 100644 plot_statruns.py diff --git a/GP_preference_demo.py b/GP_preference_demo.py index fa194bd..305dfff 100644 --- a/GP_preference_demo.py +++ b/GP_preference_demo.py @@ -10,29 +10,35 @@ plt.rc('text', usetex=True) np.random.seed(1) -train_hyper = False -use_test_data = True +train_hyper = True +use_test_data = False verbose = True #log_hyp = np.log([0.2, 0.5, 0.1, 1.0, 10.0]) # length_scale/s, sigma_f, sigma_n_abs, sigma_beta, v_beta # log_hyp = np.log([0.07, 1.0, 0.25, 1.0, 28.1]) -log_hyp = np.log([0.065, 0.8, 0.8, 0.8, 20.0]) +# log_hyp = np.log([0.065, 0.8, 0.8, 0.8, 20.0]) +log_hyp = np.log([0.05, 1.5, 0.09, 2.0, 50.0]) np.random.seed(0) -n_rel_train = 3 -n_abs_train = 1 -rel_sigma = 0.2 +n_rel_train = 30 +n_abs_train = 30 +rel_sigma = 0.02 delta_f = 1e-5 beta_sigma = 0.8 -beta_v = 20.0 +beta_v = 100.0 n_xplot = 101 n_mcsamples = 1000 n_ysamples = 101 # Define polynomial function to be modelled -true_function = test_data.zero_fun +#true_function = test_data.zero_fun +random_wave = test_data.VariableWave([0.6, 1.2], [5.0, 10.0], [0.0, 1.0], [10.0, 20.0]) +random_wave.randomize() +random_wave.set_values(a=1.2, f=6.0, o=.2, d=20.0) +true_function = random_wave.out +random_wave.print_values() rel_obs_fun = GPpref.RelObservationSampler(true_function, GPpref.PrefProbit(sigma=rel_sigma)) abs_obs_fun = GPpref.AbsObservationSampler(true_function, GPpref.AbsBoundProbit(sigma=beta_sigma, v=beta_v)) diff --git a/GPpref.py b/GPpref.py index daf542c..c5dee46 100644 --- a/GPpref.py +++ b/GPpref.py @@ -14,7 +14,7 @@ def std_norm_pdf(x): #return np.exp(-(x**2)/2)/_sqrt_2pi def std_norm_cdf(x): - #x = np.clip(x, -30, 100 ) + x = np.clip(x, -30, 100 ) return norm.cdf(x) def norm_pdf_norm_cdf_ratio(z): @@ -132,7 +132,7 @@ def log_likelihood(self, y, f): return np.log(self.likelihood(y, f)) def posterior_likelihood(self, fhat, varhat, uvi, y=1): # This is the likelihood assuming a Gaussian over f - var_star = 2*self.sigma**2 + np.atleast_2d([varhat[u, u] + varhat[v, v] - varhat[u, v] - varhat[v, v] for u,v in uvi]).T + var_star = 2*self.sigma**2 + np.atleast_2d([varhat[u, u] + varhat[v, v] - varhat[u, v] - varhat[v, u] for u,v in uvi]).T p_y = self.likelihood(y, self.get_rel_f(fhat, uvi), 1.0/np.sqrt(var_star)) return p_y @@ -353,8 +353,8 @@ def calc_laplace(self, loghyp): return f#, lml - def _safe_invert_noise(self, mat): - eps = 1.0e-6 + def _safe_invert_noise(self, mat, start_noise=1.0e-6): + eps = start_noise inv_ok = False while not inv_ok: @@ -384,8 +384,10 @@ def predict_latent(self, x): kt = self.kern.K(self.x_train_all, x) mean_latent = np.matmul(kt.T, self.iKf) Ktt = self.kern.K(x) - # iKW, _ = self._safe_invert_noise(self.KWI) - iKW = np.linalg.inv(self.KWI) + try: + iKW = np.linalg.inv(self.KWI) + except np.linalg.linalg.LinAlgError: + raise var_latent = Ktt - np.matmul(kt.T, np.matmul(iKW, np.matmul(self.W, kt))) return mean_latent, var_latent @@ -462,6 +464,13 @@ def _gen_x_obs(n, n_xdim=1, domain=None): x_test = x_test * np.diff(domain, axis=0) + domain[0, :] return x_test + def cheat_multi_sampler(self, x): + fx = np.random.normal(loc=self.f(x), scale=self.l.sigma) + max_xi = np.argmax(fx) + other_xi = np.delete(np.arange(x.shape[0]), max_xi) + y = np.ones((x.shape[0]-1, 1), dtype='int') + uvi = np.hstack((np.atleast_2d(other_xi).T, max_xi*y)) + return y, uvi, fx class AbsObservationSampler(ObservationSampler): def observation_likelihood_array(self, x, y): diff --git a/active_learners.py b/active_learners.py index fcc6880..0be43b6 100644 --- a/active_learners.py +++ b/active_learners.py @@ -3,8 +3,14 @@ from scipy.stats import beta import plot_tools as ptt -def calc_ucb(fhat, vhat, gamma=2.0): - return fhat + gamma * np.sqrt(np.atleast_2d(vhat.diagonal()).T) + +def calc_ucb(fhat, vhat, gamma=2.0, sigma_offset=0.0): + return fhat + gamma * (np.sqrt(np.atleast_2d(vhat.diagonal()).T) - sigma_offset) + +def softmax_selector(x, tau=1.0): + ex = np.exp((x - x.max())/tau) + Px = ex/ex.sum() + return np.random.choice(len(x), p=Px) class ActiveLearner(GPpref.PreferenceGaussianProcess): def init_extras(self): @@ -24,11 +30,10 @@ def solve_laplace(self, log_hyp=None): def get_observations(self): return self.x_rel, self.uvi_rel, self.x_abs, self.y_rel, self.y_abs - def select_observation(self, domain=None): - if np.random.uniform() < 0.5: - return self.uniform_domain_sampler(1, domain), None - else: - return self.uniform_domain_sampler(2, domain), np.array([[0, 1]]) + def select_observation(self, p_rel=0.5, domain=None, n_rel_samples=2): + if np.random.uniform() > p_rel: # i.e choose an absolute sample + n_rel_samples = 1 + return self.uniform_domain_sampler(n_rel_samples, domain) def uniform_domain_sampler(self, n_samples, domain=None): # Domain should be 2 x n_xdim, i.e [[x0_lo, x1_lo, ... , xn_lo], [x0_hi, x1_hi, ... , xn_hi ]] @@ -62,19 +67,48 @@ def create_posterior_plot(self, x_test, f_true, mu_true, rel_sigma, fuv_train, a class UCBLatent(ActiveLearner): # All absolute returns - def select_observation(self, domain=None, ntest=100, gamma=2.0): - x_test = self.uniform_domain_sampler(ntest, domain) + def select_observation(self, domain=None, n_test=100, gamma=2.0): + x_test = self.uniform_domain_sampler(n_test, domain) fhat, vhat = self.predict_latent(x_test) ucb = calc_ucb(fhat, vhat, gamma) - return x_test[np.argmax(ucb), :], None + return x_test[[np.argmax(ucb)], :] class UCBOut(ActiveLearner): - def select_observation(self, domain=None, ntest=100, gamma=2.0): + def select_observation(self, domain=None, n_test=100, gamma=2.0): # Don't know how to recover the second moment of the predictive distribution, so this isn't done - x_test = self.uniform_domain_sampler(ntest, domain) + x_test = self.uniform_domain_sampler(n_test, domain) fhat, vhat = self.predict_latent(x_test) Ey = self.expected_y(x_test, fhat, vhat) - return x_test[np.argmax(Ey), :], None + return x_test[[np.argmax(Ey)], :] + +class ABSThresh(ActiveLearner): + def select_observation(self, domain=None, n_test=100, p_thresh=0.7): + x_test = self.uniform_domain_sampler(n_test, domain) + fhat, vhat = self.predict_latent(x_test) + aa, bb = self.abs_likelihood.get_alpha_beta(fhat) + p_under_thresh = beta.cdf(p_thresh, aa, bb) + # ucb = calc_ucb(fhat, vhat, gamma) + return x_test[[np.argmax(p_under_thresh * (1.0 - p_under_thresh))], :] + +class UCBAbsRel(ActiveLearner): + def select_observation(self, domain=None, n_test=100, p_rel=0.5, n_rel_samples=2, gamma=2.0, tau=5.0): + x_test = self.uniform_domain_sampler(n_test, domain) + fhat, vhat = self.predict_latent(x_test) + ucb = calc_ucb(fhat, vhat, gamma).flatten() + + if np.random.uniform() < p_rel: # i.e choose a relative sample + best_n = [softmax_selector(ucb, tau=tau)] #[np.argmax(ucb)] # + # p_rel_y = self.rel_posterior_likelihood_array(fhat=fhat, varhat=vhat) + sq_dist = GPpref.squared_distance(x_test, x_test) + while len(best_n) < n_rel_samples: + # ucb = ucb*sq_dist[best_n[-1], :] # Discount ucb by distance + ucb[best_n[-1]] = 0.0 + # ucb /= p_rel_y[best_n[-1],:] # Divide by likelihood that each point is better than previous best + best_n.append(softmax_selector(ucb, tau=tau*5.0)) + # best_n.append(np.argmax(ucb)) + else: + best_n = [np.argmax(ucb)] # [softmax_selector(ucb, tau=tau)] # + return x_test[best_n, :] class PeakComparitor(ActiveLearner): @@ -96,7 +130,8 @@ def reset_observations(self): except AttributeError: print "reset_observations failed: existing observations not found" - def select_observation(self, domain=None, n_test=50, gamma=2.0, n_comparators=1): + def select_observation(self, domain=None, n_test=50, gamma=2.0, n_rel_samples=2): + n_comparators = n_rel_samples-1 x_test = self.uniform_domain_sampler(n_test, domain) fhat, vhat = self.predict_latent(x_test) ucb = calc_ucb(fhat, vhat, gamma) @@ -113,21 +148,20 @@ def select_observation(self, domain=None, n_test=50, gamma=2.0, n_comparators=1) for i,uvi1 in enumerate(other_xi): x[1] = x_test[uvi1] V[i] += p_pref[i]*self.test_observation(x, self._minus_y_obs, x_test, gamma) - V[i] += (1-p_pref[i])*self.test_observation(x, self._plus_y_obs, x_test, gamma) + if (1 - p_pref[i]) > 1e-3: + V[i] += (1-p_pref[i])*self.test_observation(x, self._plus_y_obs, x_test, gamma) best_n = np.argpartition(V, -n_comparators)[-n_comparators:] # best = np.argmax(V) cVmax = np.argmax(ucb) # This is repeated in case I want to change max_xi if ucb[cVmax] > V.max(): - return x_test[[cVmax], :], None + return x_test[[cVmax], :] else: xi = np.zeros(n_comparators+1, dtype='int') xi[0] = max_xi - xi[1:] = best_n - uvi_out = np.zeros(shape=(n_comparators, 2), dtype='int') - uvi_out[:, 1] = np.arange(start=1, stop=n_comparators+1, dtype='int') - return x_test[xi,:], uvi_out + xi[1:] = other_xi[best_n] + return x_test[xi, :] class LikelihoodImprovement(PeakComparitor): @@ -142,7 +176,8 @@ def test_observation(self, x, y, x_test, max_xi): self.reset_observations() return p_new_is_better - def select_observation(self, domain=None, n_test=50, req_improvement=0.6, n_comparators=1, gamma=1.5): + def select_observation(self, domain=None, n_test=50, req_improvement=0.6, n_rel_samples=2, gamma=1.5, p_thresh=0.7): + n_comparators = n_rel_samples-1 x_test = self.uniform_domain_sampler(n_test, domain) fhat, vhat = self.predict_latent(x_test) max_xi = np.argmax(fhat) @@ -159,21 +194,29 @@ def select_observation(self, domain=None, n_test=50, req_improvement=0.6, n_comp for i,uvi1 in enumerate(other_xi): x[1] = x_test[uvi1] V[i] += p_pref[i]*self.test_observation(x, self._minus_y_obs, x_test, max_xi) - V[i] += (1-p_pref[i])*self.test_observation(x, self._plus_y_obs, x_test, max_xi) + if (1-p_pref[i]) > 1e-3: + V[i] += (1-p_pref[i])*self.test_observation(x, self._plus_y_obs, x_test, max_xi) - best_n = np.argpartition(V, -n_comparators)[-n_comparators:] + Vmax = V.max() + # best_n = np.argpartition(V, -n_comparators)[-n_comparators:] # best = np.argmax(V) - print 'V_max = {0}'.format(V.max()) - - if V.max() < req_improvement: - ucb = calc_ucb(fhat, vhat, gamma) - return x_test[[np.argmax(ucb)], :], None + print 'V_max = {0}'.format(Vmax) + + if Vmax < req_improvement: + # aa, bb = self.abs_likelihood.get_alpha_beta(fhat) + # p_under_thresh = beta.cdf(p_thresh, aa, bb) + # return x_test[[np.argmax(p_under_thresh*(1.0-p_under_thresh))], :] + ucb = calc_ucb(fhat, vhat, gamma, self.rel_likelihood.sigma) + return x_test[[np.argmax(ucb)], :] else: + best_n = [] + while len(best_n) < n_comparators: + cbest = np.argmax(V) + best_n.append(cbest) + V = V * np.sqrt(GPpref.squared_distance(x_test[[other_xi[cbest]], :], x_test[other_xi])[0]) xi = np.zeros(n_comparators+1, dtype='int') xi[0] = max_xi - xi[1:] = best_n - uvi_out = np.zeros(shape=(n_comparators, 2), dtype='int') - uvi_out[:, 1] = np.arange(start=1, stop=n_comparators+1, dtype='int') - return x_test[xi,:], uvi_out + xi[1:] = other_xi[best_n] + return x_test[xi, :] diff --git a/active_statruns.py b/active_statruns.py new file mode 100644 index 0000000..51566ce --- /dev/null +++ b/active_statruns.py @@ -0,0 +1,168 @@ +# Simple 1D GP classification example +import time +import numpy as np +import matplotlib.pyplot as plt +import GPpref +import plot_tools as ptt +from active_learners import ActiveLearner, UCBLatent, PeakComparitor, LikelihoodImprovement, ABSThresh, UCBAbsRel +import test_data +import pickle + +class Learner(object): + def __init__(self, model_type, obs_arguments): + self.model_type = model_type + self.obs_arguments = obs_arguments + + def build_model(self, training_data): + self.model = self.model_type(**training_data) + +def wrms(y_true, y_est, weight=True): + if weight: + w = y_true + else: + w = 1.0 + return np.sqrt(np.mean(((y_true - y_est)*w)**2)) + +nowstr = time.strftime("%Y_%m_%d-%H_%M") +plt.rc('font',**{'family':'serif','sans-serif':['Computer Modern Roman']}) +plt.rc('text', usetex=True) + +# log_hyp = np.log([0.1,0.5,0.1,10.0]) # length_scale, sigma_f, sigma_probit, v_beta +# log_hyp = np.log([0.07, 0.75, 0.25, 1.0, 28.1]) +log_hyp = np.log([0.05, 1.5, 0.09, 2.0, 50.0]) +np.random.seed(10) + +n_rel_train = 1 +n_abs_train = 0 +rel_sigma = 0.02 +delta_f = 1e-5 + +beta_sigma = 0.8 +beta_v = 100.0 + +n_xtest = 101 +n_best_points = 15 + +n_mcsamples = 1000 +n_ysamples = 101 +n_trials = 100 +n_rel_samples = 5 + +n_queries = 20 + +# Define polynomial function to be modelled +random_wave = test_data.VariableWave([0.6, 1.0], [5.0, 10.0], [0.0, 1.0], [10.0, 20.0]) + +nowstr = time.strftime("%Y_%m_%d-%H_%M") +data_dir = 'data/' + nowstr + '/' +ptt.ensure_dir(data_dir) +print "Data will be saved to: {0}".format(data_dir) + + +# True function +x_plot = np.linspace(0.0,1.0,n_xtest,dtype='float') +x_test = np.atleast_2d(x_plot).T + +# Construct active learner object +learners = [Learner(ActiveLearner, {'p_rel': 0.5, 'n_rel_samples': n_rel_samples}), # 'Random (rel and abs)', + Learner(ActiveLearner, {'p_rel': 1.0, 'n_rel_samples': n_rel_samples}), # 'Random (rel)', + Learner(ActiveLearner, {'p_rel': 0.0, 'n_rel_samples': n_rel_samples}), # 'Random (abs)', + Learner(UCBLatent, {'gamma': 2.0, 'n_test': 100}), # 'UCBLatent' + Learner(UCBAbsRel, { 'n_test': 100, 'p_rel': 0.5, 'n_rel_samples': n_rel_samples, 'gamma': 2.0, 'tau':5.0}), # 'UCBCombined', + # Learner(ABSThresh, {'n_test': 100, 'p_thresh': 0.7}), # 'ABSThresh' + # Learner(PeakComparitor, {'gamma': 2.0, 'n_test': 50, 'n_rel_samples': n_rel_samples}), # 'PeakComparitor' + # Learner(LikelihoodImprovement, {'req_improvement': 0.60, 'n_test': 50, 'gamma': 2.0, 'n_rel_samples': n_rel_samples, 'p_thresh': 0.7}) # 'LikelihoodImprovement' + ] +names = ['Random (rel and abs)', 'Random (rel)', 'Random (abs)', 'UCBLatent (abs)', 'UCBCombined (rel and abs)'] +n_learners = len(learners) + +obs_array = [{'name': name, 'obs': []} for name in names] + +wrms_results = np.zeros((n_learners, n_queries+1, n_trials)) +true_pos_results = np.zeros((n_learners, n_queries+1, n_trials), dtype='int') +selected_error = np.zeros((n_learners, n_queries+1, n_trials)) + +for trial_number in range(n_trials): + print 'Trial {0}'.format(trial_number) + random_wave.randomize(print_vals=True) + rel_obs_fun = GPpref.RelObservationSampler(random_wave.out, GPpref.PrefProbit(sigma=rel_sigma)) + abs_obs_fun = GPpref.AbsObservationSampler(random_wave.out, GPpref.AbsBoundProbit(sigma=beta_sigma, v=beta_v)) + f_true = abs_obs_fun.f(x_test) + y_abs_true = abs_obs_fun.mean_link(x_test) + best_points = np.argpartition(y_abs_true.flatten(), -n_best_points)[-n_best_points:] + best_points_set = set(best_points) + abs_y_samples = np.atleast_2d(np.linspace(0.01, 0.99, n_ysamples)).T + p_abs_y_true = abs_obs_fun.observation_likelihood_array(x_test, abs_y_samples) + p_rel_y_true = rel_obs_fun.observation_likelihood_array(x_test) + + # Initial data + x_rel, uvi_rel, uv_rel, y_rel, fuv_rel = rel_obs_fun.generate_n_observations(n_rel_train, n_xdim=1) + x_abs, y_abs, mu_abs = abs_obs_fun.generate_n_observations(n_abs_train, n_xdim=1) + training_data = {'x_rel': x_rel, 'uvi_rel': uvi_rel, 'x_abs': x_abs, 'y_rel': y_rel, 'y_abs': y_abs, + 'delta_f': delta_f, 'rel_likelihood': GPpref.PrefProbit(), + 'abs_likelihood': GPpref.AbsBoundProbit()} + + # Get initial solution + for nl, learner in enumerate(learners): + learner.build_model(training_data) + learner.model.set_hyperparameters(log_hyp) + f = learner.model.solve_laplace() + fhat, vhat = learner.model.predict_latent(x_test) + y_abs_est = learner.model.abs_posterior_mean(x_test, fhat, vhat) + wrms_results[nl, 0, trial_number] = wrms(y_abs_true, y_abs_est) + + for obs_num in range(n_queries): + learners[4].obs_arguments['p_rel'] = max(0.0, (20-obs_num)/20.0) + for nl, learner in enumerate(learners): + next_x = learner.model.select_observation(**learner.obs_arguments) + if next_x.shape[0] == 1: + next_y, next_f = abs_obs_fun.generate_observations(next_x) + learner.model.add_observations(next_x, next_y) + # print 'Abs: x:{0}, y:{1}'.format(next_x[0], next_y[0]) + else: + next_y, next_uvi, next_fx = rel_obs_fun.cheat_multi_sampler(next_x) + next_fuv = next_fx[next_uvi][:,:,0] + fuv_rel = np.concatenate((fuv_rel, next_fuv), 0) + learner.model.add_observations(next_x, next_y, next_uvi) + # print 'Rel: x:{0}, best_index:{1}'.format(next_x.flatten(), next_uvi[0, 1]) + f = learner.model.solve_laplace() + fhat, vhat = learner.model.predict_latent(x_test) + y_abs_est = learner.model.abs_posterior_mean(x_test, fhat, vhat) + + best_points_est = set(np.argpartition(y_abs_est.flatten(), -n_best_points)[-n_best_points:]) + true_pos_results[nl, obs_num+1, trial_number] = len(best_points_set.intersection(best_points_est)) + wrms_results[nl, obs_num+1, trial_number] = wrms(y_abs_true, y_abs_est) + selected_error[nl, obs_num + 1, trial_number] = wrms(y_abs_true[best_points], y_abs_est[best_points], weight=False) + + + print true_pos_results[:, obs_num+1, trial_number] + print wrms_results[:, obs_num+1, trial_number] + for nl, learner in enumerate(learners): + obs_tuple = learner.model.get_observations() + obs_array[nl]['obs'].append(ObsObject(*obs_tuple)) + +with open(data_dir+'wrms.pkl', 'wb') as fh: + pickle.dump(wrms_results, fh) + +with open(data_dir+'true_pos.pkl', 'wb') as fh: + pickle.dump(true_pos_results, fh) + +with open(data_dir+'selected_error.pkl', 'wb') as fh: + pickle.dump(selected_error, fh) + +with open(data_dir+'obs.pkl', 'wb') as fh: + pickle.dump(obs_array, fh) + +f0, ax0 = plt.subplots() +hl = ax0.plot(np.arange(n_queries+1), np.mean(wrms_results, axis=2).T) +f0.legend(hl, names) + +f1, ax1 = plt.subplots() +hl1 = ax1.plot(np.arange(n_queries+1), np.mean(true_pos_results, axis=2).T) +f1.legend(hl1, names) + +f2, ax2 = plt.subplots() +hl2 = ax2.plot(np.arange(n_queries+1), np.mean(selected_error, axis=2).T) +f2.legend(hl2, names) + +plt.show() \ No newline at end of file diff --git a/plot_statruns.py b/plot_statruns.py new file mode 100644 index 0000000..db9bb11 --- /dev/null +++ b/plot_statruns.py @@ -0,0 +1,50 @@ +import numpy as np +import matplotlib.pyplot as plt +import pickle +from Tkinter import Tk +from tkFileDialog import askopenfilename, askdirectory +from test_data import ObsObject + +Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing +data_dir = askdirectory() # show an "Open" dialog box and return the path to the selected file + +with open(data_dir+'/wrms.pkl', 'rb') as fh: + wrms_results = pickle.load(fh) # Dimensions n_learners, n_queries+1, n_trials + +with open(data_dir+'/true_pos.pkl', 'rb') as fh: + true_pos_results = pickle.load(fh) + +with open(data_dir+'/selected_error.pkl', 'rb') as fh: + selected_error = pickle.load(fh) + +with open(data_dir+'/obs.pkl', 'rb') as fh: + obs_array = pickle.load(fh) + + +names = [l['name'] for l in obs_array] +n_queries = wrms_results.shape[1]-1 +mean_err = np.mean(wrms_results, axis=2) +std_err = np.std(wrms_results, axis=2) + +f0, ax0 = plt.subplots() +hl = ax0.plot(np.arange(n_queries+1), np.mean(wrms_results, axis=2).T) +f0.legend(hl, names) +ax0.set_title('Weighted RMSE') + +f1, ax1 = plt.subplots() +hl1 = ax1.plot(np.arange(n_queries+1), np.mean(true_pos_results, axis=2).T/15.0) +f1.legend(hl1, names) +ax1.set_title('True positive selections') + +f2, ax2 = plt.subplots() +hl2 = ax2.plot(np.arange(n_queries+1), np.mean(selected_error, axis=2).T) +f2.legend(hl2, names) +ax2.set_title('RMSE of best paths') + + +# hl = [] +# for i in range(mean_err.shape[0]): +# hl.append(plt.errorbar(np.arange(mean_err.shape[1]), mean_err[i,:], yerr=std_err[i, :])) +# plt.legend(hl, names) + +plt.show() \ No newline at end of file diff --git a/pref_active_learning.py b/pref_active_learning.py index c2b3d94..e41671f 100644 --- a/pref_active_learning.py +++ b/pref_active_learning.py @@ -14,16 +14,17 @@ save_plots = True # log_hyp = np.log([0.1,0.5,0.1,10.0]) # length_scale, sigma_f, sigma_probit, v_beta -log_hyp = np.log([0.07, 0.75, 0.25, 1.0, 28.1]) +# log_hyp = np.log([0.07, 0.75, 0.25, 1.0, 28.1]) +log_hyp = np.log([0.05, 1.5, 0.09, 2.0, 50.0]) np.random.seed(0) n_rel_train = 1 n_abs_train = 1 -rel_sigma = 0.2 +rel_sigma = 0.02 delta_f = 1e-5 beta_sigma = 0.8 -beta_v = 20.0 +beta_v = 100.0 n_xplot = 101 n_mcsamples = 1000 @@ -32,7 +33,12 @@ n_queries = 20 # Define polynomial function to be modelled -true_function = test_data.multi_peak +# true_function = test_data.multi_peak +random_wave = test_data.VariableWave([0.6, 1.0], [5.0, 10.0], [0.0, 1.0], [10.0, 20.0]) +random_wave.randomize() +random_wave.set_values(1.0, 6.00, 0.2, 10.50) +true_function = random_wave.out +random_wave.print_values() if save_plots: nowstr = time.strftime("%Y_%m_%d-%H_%M") @@ -67,9 +73,10 @@ fig_t.savefig(fig_dir+'true.pdf', bbox_inches='tight') # Construct active learner object -learner = active_learners.LikelihoodImprovement(x_rel, uvi_rel, x_abs, y_rel, y_abs, delta_f=delta_f, +learner = active_learners.UCBAbsRel(x_rel, uvi_rel, x_abs, y_rel, y_abs, delta_f=delta_f, rel_likelihood=GPpref.PrefProbit(), abs_likelihood=GPpref.AbsBoundProbit()) - +# obs_arguments = {'req_improvement': 0.60, 'n_test': 50, 'gamma': 2.0, 'n_rel_samples': 5, 'p_thresh': 0.7} +obs_arguments = {'n_test': 100, 'p_rel': 0.5, 'n_rel_samples': 5, 'gamma': 2.0} # Get initial solution learner.set_hyperparameters(log_hyp) f = learner.solve_laplace() @@ -80,15 +87,18 @@ fig_p.savefig(fig_dir+'posterior00.pdf', bbox_inches='tight') for obs_num in range(n_queries): - next_x, next_uvi = learner.select_observation(req_improvement=0.55, gamma=2.0, n_comparators=4) - if next_uvi is None: + next_x = learner.select_observation(**obs_arguments) + if next_x.shape[0] == 1: next_y, next_f = abs_obs_fun.generate_observations(next_x) learner.add_observations(next_x, next_y) + print 'Abs: x:{0}, y:{1}'.format(next_x[0], next_y[0]) else: - next_y, next_f = rel_obs_fun.generate_observations(next_x[next_uvi][:,:,0]) - fuv_rel = np.concatenate((fuv_rel, next_f), 0) + next_y, next_uvi, next_fx = rel_obs_fun.cheat_multi_sampler(next_x) + next_fuv = next_fx[next_uvi][:, :, 0] + fuv_rel = np.concatenate((fuv_rel, next_fuv), 0) learner.add_observations(next_x, next_y, next_uvi) - print next_x, next_y + print 'Rel: x:{0}, best_index:{1}'.format(next_x.flatten(), next_uvi[0, 1]) + f = learner.solve_laplace() if save_plots: fig_p, (ax_p_l, ax_p_a, ax_p_r) = learner.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, diff --git a/test_data.py b/test_data.py index 5116973..5340fcd 100644 --- a/test_data.py +++ b/test_data.py @@ -1,6 +1,41 @@ import numpy as np +class ObsObject(object): + def __init__(self, x_rel, uvi_rel, x_abs, y_rel, y_abs): + self.x_rel, self.uvi_rel, self.x_abs, self.y_rel, self.y_abs = x_rel, uvi_rel, x_abs, y_rel, y_abs + + +class VariableWave(object): + def __init__(self, amp_range, f_range, off_range, damp_range, n_components=1): + self.amp_range = amp_range + self.f_range = f_range + self.off_range = off_range + self.damp_range = damp_range + self.n_components = n_components + self.randomize() + + def out(self, x): + y = self.amplitude*np.cos(self.frequency * np.pi * (x-self.offset)) * np.exp(-self.damping*(x-self.offset)**2) + return y + + def set_values(self, a, f, o, d): + self.amplitude = a + self.frequency = f + self.offset = o + self.damping = d + + def randomize(self, print_vals=False): + self.amplitude = np.random.uniform(low=self.amp_range[0], high=self.amp_range[1]/self.n_components) + self.frequency = np.random.uniform(low=self.f_range[0], high=self.f_range[1]) + self.offset = np.random.uniform(low=self.off_range[0], high=self.off_range[1]) + self.damping = np.random.uniform(low=self.damp_range[0], high=self.damp_range[1]) + if print_vals: + self.print_values() + + def print_values(self): + print "a: {0:.2f}, f: {1:.2f}, o: {2:.2f}, d: {3:.2f}".format(self.amplitude, self.frequency, self.offset, self.damping) + def damped_wave(x): y = np.cos(6 * np.pi * (x - 0.5)) * np.exp(-10 * (x - 0.5) ** 2) return y From 2d5a66b0e7a192dd072d8dfe1e64fcb467650412 Mon Sep 17 00:00:00 2001 From: Nicholas Lawrance Date: Tue, 2 May 2017 18:12:45 -0700 Subject: [PATCH 12/38] Rarer events in statruns, added error bars to plots --- GP_preference_demo.py | 32 ++++++++++------ GPpref.py | 2 +- active_learners.py | 1 + active_statruns.py | 13 ++++--- plot_statruns.py | 83 +++++++++++++++++++++++++---------------- plot_tools.py | 62 ++++++++++++++++++++++++++++++ pref_active_learning.py | 25 +++++++++---- test_data.py | 19 ++++++++++ 8 files changed, 178 insertions(+), 59 deletions(-) diff --git a/GP_preference_demo.py b/GP_preference_demo.py index 305dfff..34688a1 100644 --- a/GP_preference_demo.py +++ b/GP_preference_demo.py @@ -8,25 +8,26 @@ # from scipy.stats import beta plt.rc('font',**{'family':'serif','sans-serif':['Computer Modern Roman']}) plt.rc('text', usetex=True) -np.random.seed(1) +np.random.seed(0) -train_hyper = True +train_hyper = False use_test_data = False verbose = True #log_hyp = np.log([0.2, 0.5, 0.1, 1.0, 10.0]) # length_scale/s, sigma_f, sigma_n_abs, sigma_beta, v_beta # log_hyp = np.log([0.07, 1.0, 0.25, 1.0, 28.1]) # log_hyp = np.log([0.065, 0.8, 0.8, 0.8, 20.0]) -log_hyp = np.log([0.05, 1.5, 0.09, 2.0, 50.0]) -np.random.seed(0) +# log_hyp = np.log([0.05, 1.5, 0.09, 2.0, 50.0]) +log_hyp = np.log([0.02, 0.6, 0.2, 0.8, 60.0]) -n_rel_train = 30 -n_abs_train = 30 -rel_sigma = 0.02 +n_rel_train = 100 +n_abs_train = 100 + +rel_sigma = 0.05 delta_f = 1e-5 beta_sigma = 0.8 -beta_v = 100.0 +beta_v = 80.0 n_xplot = 101 n_mcsamples = 1000 @@ -34,11 +35,18 @@ # Define polynomial function to be modelled #true_function = test_data.zero_fun -random_wave = test_data.VariableWave([0.6, 1.2], [5.0, 10.0], [0.0, 1.0], [10.0, 20.0]) -random_wave.randomize() -random_wave.set_values(a=1.2, f=6.0, o=.2, d=20.0) -true_function = random_wave.out + +# random_wave = test_data.VariableWave(amp_range=[0.6, 1.2], f_range=[5.0, 10.0], off_range=[0.2, 0.8], +# damp_range=[30.0, 100.0]) +# #random_wave.set_values(a=1.2, f=6.0, o=.2, d=20.0) +# random_wave.print_values() + +random_wave = test_data.MultiWave(amp_range=[0.6, 1.2], f_range=[10.0, 30.0], off_range=[0.1, 0.9], + damp_range=[250.0, 350.0], n_components=3) random_wave.print_values() +true_function = random_wave.out + +# true_function = test_data.damped_wave rel_obs_fun = GPpref.RelObservationSampler(true_function, GPpref.PrefProbit(sigma=rel_sigma)) abs_obs_fun = GPpref.AbsObservationSampler(true_function, GPpref.AbsBoundProbit(sigma=beta_sigma, v=beta_v)) diff --git a/GPpref.py b/GPpref.py index c5dee46..d83a111 100644 --- a/GPpref.py +++ b/GPpref.py @@ -219,7 +219,7 @@ def generate_samples(self, f): class PreferenceGaussianProcess(object): - def __init__(self, x_rel, uvi_rel, x_abs, y_rel, y_abs, rel_likelihood=PrefProbit(), delta_f = 1e-6, + def __init__(self, x_rel, uvi_rel, x_abs, y_rel, y_abs, rel_likelihood=PrefProbit(), delta_f=1e-6, abs_likelihood=AbsBoundProbit(), verbose=False): # log_hyp are log of hyperparameters, note that it is [length_0, ..., length_d, sigma_f, sigma_probit, v_beta] # Training points are split into relative and absolute for calculating f, but combined for predictions. diff --git a/active_learners.py b/active_learners.py index 0be43b6..44d0d2a 100644 --- a/active_learners.py +++ b/active_learners.py @@ -74,6 +74,7 @@ def select_observation(self, domain=None, n_test=100, gamma=2.0): return x_test[[np.argmax(ucb)], :] class UCBOut(ActiveLearner): + # NOT FULLY IMPLEMENTED - BROKEN def select_observation(self, domain=None, n_test=100, gamma=2.0): # Don't know how to recover the second moment of the predictive distribution, so this isn't done x_test = self.uniform_domain_sampler(n_test, domain) diff --git a/active_statruns.py b/active_statruns.py index 51566ce..923988c 100644 --- a/active_statruns.py +++ b/active_statruns.py @@ -29,16 +29,17 @@ def wrms(y_true, y_est, weight=True): # log_hyp = np.log([0.1,0.5,0.1,10.0]) # length_scale, sigma_f, sigma_probit, v_beta # log_hyp = np.log([0.07, 0.75, 0.25, 1.0, 28.1]) -log_hyp = np.log([0.05, 1.5, 0.09, 2.0, 50.0]) +# log_hyp = np.log([0.05, 1.5, 0.09, 2.0, 50.0]) +log_hyp = np.log([0.02, 0.6, 0.2, 0.8, 60.0]) np.random.seed(10) n_rel_train = 1 n_abs_train = 0 -rel_sigma = 0.02 +rel_sigma = 0.05 delta_f = 1e-5 beta_sigma = 0.8 -beta_v = 100.0 +beta_v = 80.0 n_xtest = 101 n_best_points = 15 @@ -51,7 +52,9 @@ def wrms(y_true, y_est, weight=True): n_queries = 20 # Define polynomial function to be modelled -random_wave = test_data.VariableWave([0.6, 1.0], [5.0, 10.0], [0.0, 1.0], [10.0, 20.0]) +# random_wave = test_data.VariableWave([0.6, 1.0], [5.0, 10.0], [0.0, 1.0], [10.0, 20.0]) +random_wave = test_data.MultiWave(amp_range=[0.6, 1.2], f_range=[10.0, 30.0], off_range=[0.1, 0.9], + damp_range=[250.0, 350.0], n_components=3) nowstr = time.strftime("%Y_%m_%d-%H_%M") data_dir = 'data/' + nowstr + '/' @@ -139,7 +142,7 @@ def wrms(y_true, y_est, weight=True): print wrms_results[:, obs_num+1, trial_number] for nl, learner in enumerate(learners): obs_tuple = learner.model.get_observations() - obs_array[nl]['obs'].append(ObsObject(*obs_tuple)) + obs_array[nl]['obs'].append(test_data.ObsObject(*obs_tuple)) with open(data_dir+'wrms.pkl', 'wb') as fh: pickle.dump(wrms_results, fh) diff --git a/plot_statruns.py b/plot_statruns.py index db9bb11..26ad50f 100644 --- a/plot_statruns.py +++ b/plot_statruns.py @@ -2,49 +2,66 @@ import matplotlib.pyplot as plt import pickle from Tkinter import Tk -from tkFileDialog import askopenfilename, askdirectory -from test_data import ObsObject +from tkFileDialog import askdirectory +# from test_data import ObsObject -Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing -data_dir = askdirectory() # show an "Open" dialog box and return the path to the selected file -with open(data_dir+'/wrms.pkl', 'rb') as fh: - wrms_results = pickle.load(fh) # Dimensions n_learners, n_queries+1, n_trials +def single_plot_SEM(data, x=None, names=None, title=None, xlabel='Number of samples', bars=False): + mean_err = np.mean(data, axis=2) + std_err_mean = np.std(data, axis=2, ddof=1) + if x is None: + x = np.arange(data.shape[1]) -with open(data_dir+'/true_pos.pkl', 'rb') as fh: - true_pos_results = pickle.load(fh) + hf, hax = plt.subplots() + hl = [] + for mu, sig in zip(mean_err, std_err_mean): + if bars: + hl.append(hax.errorbar(x, mu, yerr=sig, capsize=2.0)) + else: + hl.append(hax.plot(x, mu)[0]) + hax.legend(hl, names, loc='best') + hax.set_title(title) + hax.set_xlabel(xlabel) + return hf, hax -with open(data_dir+'/selected_error.pkl', 'rb') as fh: - selected_error = pickle.load(fh) -with open(data_dir+'/obs.pkl', 'rb') as fh: - obs_array = pickle.load(fh) +def plot_statrun_results(wrms_results, true_pos_results, selected_error, obs_array, data_dir=None, bars=False): + names = [l['name'] for l in obs_array] + f0, ax0 = single_plot_SEM(wrms_results, names=names, title='Weighted RMSE', bars=bars) + f1, ax1 = single_plot_SEM(true_pos_results, names=names, title='True positive selections', bars=False) + f2, ax2 = single_plot_SEM(selected_error, names=names, title='RMSE of best paths', bars=False) -names = [l['name'] for l in obs_array] -n_queries = wrms_results.shape[1]-1 -mean_err = np.mean(wrms_results, axis=2) -std_err = np.std(wrms_results, axis=2) + if data_dir is not None: + f0.savefig(data_dir + '/wrms.pdf', bbox_inches='tight') + f1.savefig(data_dir + '/true_pos.pdf', bbox_inches='tight') + f2.savefig(data_dir + '/rms_best.pdf', bbox_inches='tight') + # hl = [] + # for i in range(mean_err.shape[0]): + # hl.append(plt.errorbar(np.arange(mean_err.shape[1]), mean_err[i,:], yerr=std_err[i, :])) + # plt.legend(hl, names) + return f0, f1, f2 -f0, ax0 = plt.subplots() -hl = ax0.plot(np.arange(n_queries+1), np.mean(wrms_results, axis=2).T) -f0.legend(hl, names) -ax0.set_title('Weighted RMSE') +def load_and_plot(save_plots=True, *args, **kwargs): + Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing + data_dir = askdirectory() # show an "Open" dialog box and return the path to the selected file -f1, ax1 = plt.subplots() -hl1 = ax1.plot(np.arange(n_queries+1), np.mean(true_pos_results, axis=2).T/15.0) -f1.legend(hl1, names) -ax1.set_title('True positive selections') + with open(data_dir+'/wrms.pkl', 'rb') as fh: + wrms_results = pickle.load(fh) # Dimensions n_learners, n_queries+1, n_trials -f2, ax2 = plt.subplots() -hl2 = ax2.plot(np.arange(n_queries+1), np.mean(selected_error, axis=2).T) -f2.legend(hl2, names) -ax2.set_title('RMSE of best paths') + with open(data_dir+'/true_pos.pkl', 'rb') as fh: + true_pos_results = pickle.load(fh) + with open(data_dir+'/selected_error.pkl', 'rb') as fh: + selected_error = pickle.load(fh) -# hl = [] -# for i in range(mean_err.shape[0]): -# hl.append(plt.errorbar(np.arange(mean_err.shape[1]), mean_err[i,:], yerr=std_err[i, :])) -# plt.legend(hl, names) + with open(data_dir+'/obs.pkl', 'rb') as fh: + obs_array = pickle.load(fh) -plt.show() \ No newline at end of file + if not save_plots: + data_dir = None + plot_statrun_results(wrms_results, true_pos_results, selected_error, obs_array, data_dir=data_dir, *args, **kwargs) + plt.show() + +if __name__ == "__main__": + load_and_plot(save_plots=True, bars=True) \ No newline at end of file diff --git a/plot_tools.py b/plot_tools.py index d02f9f4..a0367cb 100644 --- a/plot_tools.py +++ b/plot_tools.py @@ -22,6 +22,21 @@ def plot_with_bounds(ax, x, y, s, c=lines[0]): ax.set_ylim(bottom = min(clim[0],xy[:,1].min()), top = max(clim[1], xy[:,1].max())) return h_fx, h_patch +def plot_setup_rel(t_l = r'Latent function, $f(x)$', t_r = r'Relative likelihood, $P(x_0 \succ x_1 | f(x_0), f(x_1))$'): + + fig, (ax_l, ax_r) = plt.subplots(1, 2) + fig.set_size_inches(9.8, 3.5) + + # Latent function + ax_l.set_title(t_l) + ax_l.set_xlabel('$x$') + ax_l.set_ylabel('$f(x)$') + + # Relative likelihood + ax_r.set_title(t_r) + ax_r.set_xlabel('$x_0$') + ax_r.set_ylabel('$x_1$') + return fig, (ax_l, ax_r) def plot_setup_2d(t_l = r'Latent function, $f(x)$', t_a = r'Absolute likelihood, $p(y | f(x))$', t_r = r'Relative likelihood, $P(x_0 \succ x_1 | f(x_0), f(x_1))$'): @@ -84,6 +99,28 @@ def true_plots(xt, ft, mu_t, rel_sigma, y_samples, p_a_y, p_r_y, xa_train, ya_tr return fig, (ax_l, ax_a, ax_r) +def true_plots_rel(xt, ft, mu_t, rel_sigma, y_samples, p_a_y, p_r_y, xa_train, ya_train, uvr_train, fuvr_train, yr_train, + class_icons=['ko', 'wo'], marker_options={'mec':'k', 'mew':0.5}, *args, **kwargs): + + # Plot true function, likelihoods and observations + fig, (ax_l, ax_r) = plot_setup_rel(**kwargs) + + # True latent + plot_with_bounds(ax_l, xt, ft, rel_sigma, c=lines[0]) + + # True absolute likelihood + + # True relative likelihood + rel_y_extent = [xt[0, 0], xt[-1, 0], xt[0, 0], xt[-1, 0]] + h_prt = plot_relative_likelihood(ax_r, p_r_y, extent=rel_y_extent) + if xt.shape[0] > 0: + for uv, fuv, y in zip(uvr_train, fuvr_train, yr_train): + ax_r.plot(uv[0], uv[1], class_icons[(y[0] + 1) / 2], **marker_options) + ax_l.plot(uv, fuv, 'b-', color=lighten(lines[0])) + ax_l.plot(uv[(y + 1) / 2], fuv[(y + 1) / 2], class_icons[(y[0] + 1) / 2], **marker_options) + return fig, (ax_l, ax_r) + + def estimate_plots(xt, ft, mu_t, fhat, vhat, E_y, rel_sigma, y_samples, p_a_y, p_r_y, xa_train, ya_train, uvr_train, fuvr_train, yr_train, class_icons = ['ko', 'wo'], marker_options = {'mec':'k', 'mew':0.5}, *args, **kwargs): @@ -117,6 +154,31 @@ def estimate_plots(xt, ft, mu_t, fhat, vhat, E_y, rel_sigma, return fig, (ax_l, ax_a, ax_r) +def estimate_plots_rel(xt, ft, mu_t, fhat, vhat, E_y, rel_sigma, + y_samples, p_a_y, p_r_y, xa_train, ya_train, uvr_train, fuvr_train, yr_train, + class_icons = ['ko', 'wo'], marker_options = {'mec':'k', 'mew':0.5}, *args, **kwargs): + fig, (ax_l, ax_r) = plot_setup_rel(**kwargs) + + # Latent function + hf, hpf = plot_with_bounds(ax_l, xt, ft, rel_sigma, c=lines[0]) + + hf_hat, hpf_hat = plot_with_bounds(ax_l, xt, fhat, np.sqrt(np.atleast_2d(vhat.diagonal()).T), c=lines[1]) + ax_l.legend([hf, hf_hat], [r'True latent function, $f(x)$', r'$\mathcal{GP}$ estimate $\hat{f}(x)$']) + + # Absolute posterior likelihood + abs_extent = [xt[0, 0], xt[-1, 0], y_samples[0, 0], y_samples[-1, 0]] + + # Relative posterior likelihood + rel_y_extent = [xt[0, 0], xt[-1, 0], xt[0, 0], xt[-1, 0]] + h_prp = plot_relative_likelihood(ax_r, p_r_y, extent=rel_y_extent) + if uvr_train.shape[0] > 0: + for uv, fuv, y in zip(uvr_train, fuvr_train, yr_train): + ax_r.plot(uv[0], uv[1], class_icons[(y[0] + 1) / 2], **marker_options) + ax_l.plot(uv, fuv, 'b-', color=lighten(lines[0])) + ax_l.plot(uv[(y + 1) / 2], fuv[(y + 1) / 2], class_icons[(y[0] + 1) / 2], **marker_options) + + return fig, (ax_l, ax_r) + def ensure_dir(file_path): directory = os.path.dirname(file_path) if not os.path.exists(directory): diff --git a/pref_active_learning.py b/pref_active_learning.py index e41671f..fbea583 100644 --- a/pref_active_learning.py +++ b/pref_active_learning.py @@ -6,6 +6,7 @@ import plot_tools as ptt import active_learners import test_data +from matplotlib.backends.backend_pdf import PdfPages nowstr = time.strftime("%Y_%m_%d-%H_%M") plt.rc('font',**{'family':'serif','sans-serif':['Computer Modern Roman']}) @@ -15,16 +16,17 @@ # log_hyp = np.log([0.1,0.5,0.1,10.0]) # length_scale, sigma_f, sigma_probit, v_beta # log_hyp = np.log([0.07, 0.75, 0.25, 1.0, 28.1]) -log_hyp = np.log([0.05, 1.5, 0.09, 2.0, 50.0]) -np.random.seed(0) +# log_hyp = np.log([0.05, 1.5, 0.09, 2.0, 50.0]) +log_hyp = np.log([0.02, 0.6, 0.2, 0.8, 60.0]) +np.random.seed(2) n_rel_train = 1 n_abs_train = 1 -rel_sigma = 0.02 +rel_sigma = 0.05 delta_f = 1e-5 beta_sigma = 0.8 -beta_v = 100.0 +beta_v = 80.0 n_xplot = 101 n_mcsamples = 1000 @@ -34,9 +36,11 @@ # Define polynomial function to be modelled # true_function = test_data.multi_peak -random_wave = test_data.VariableWave([0.6, 1.0], [5.0, 10.0], [0.0, 1.0], [10.0, 20.0]) +# random_wave = test_data.VariableWave([0.6, 1.0], [5.0, 10.0], [0.0, 1.0], [10.0, 20.0]) +random_wave = test_data.MultiWave(amp_range=[0.6, 1.2], f_range=[10.0, 30.0], off_range=[0.1, 0.9], + damp_range=[250.0, 350.0], n_components=3) random_wave.randomize() -random_wave.set_values(1.0, 6.00, 0.2, 10.50) +# random_wave.set_values(1.0, 6.00, 0.2, 10.50) true_function = random_wave.out random_wave.print_values() @@ -45,6 +49,7 @@ fig_dir = 'fig/' + nowstr + '/' ptt.ensure_dir(fig_dir) print "Figures will be saved to: {0}".format(fig_dir) + pdf_pages = PdfPages(fig_dir+'posterior_all.pdf') rel_obs_fun = GPpref.RelObservationSampler(true_function, GPpref.PrefProbit(sigma=rel_sigma)) abs_obs_fun = GPpref.AbsObservationSampler(true_function, GPpref.AbsBoundProbit(sigma=beta_sigma, v=beta_v)) @@ -84,7 +89,8 @@ if save_plots: fig_p, (ax_p_l, ax_p_a, ax_p_r) = learner.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, abs_y_samples, mc_samples) - fig_p.savefig(fig_dir+'posterior00.pdf', bbox_inches='tight') + pdf_pages.savefig(fig_p, bbox_inches='tight') + # fig_p.savefig(fig_dir+'posterior00.pdf', bbox_inches='tight') for obs_num in range(n_queries): next_x = learner.select_observation(**obs_arguments) @@ -103,7 +109,8 @@ if save_plots: fig_p, (ax_p_l, ax_p_a, ax_p_r) = learner.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, abs_y_samples, mc_samples) - fig_p.savefig(fig_dir+'posterior{0:02d}.pdf'.format(obs_num+1), bbox_inches='tight') + pdf_pages.savefig(fig_p, bbox_inches='tight') + # fig_p.savefig(fig_dir+'posterior{0:02d}.pdf'.format(obs_num+1), bbox_inches='tight') plt.close(fig_p) learner.print_hyperparameters() @@ -111,5 +118,7 @@ if not save_plots: fig_p, (ax_p_l, ax_p_a, ax_p_r) = learner.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, abs_y_samples, mc_samples) +else: + pdf_pages.close() plt.show() diff --git a/test_data.py b/test_data.py index 5340fcd..09af4b5 100644 --- a/test_data.py +++ b/test_data.py @@ -36,6 +36,25 @@ def randomize(self, print_vals=False): def print_values(self): print "a: {0:.2f}, f: {1:.2f}, o: {2:.2f}, d: {3:.2f}".format(self.amplitude, self.frequency, self.offset, self.damping) + +class MultiWave(VariableWave): + def out(self, x): + y = np.zeros(np.array(x).shape) + for a, f, o, d in zip(self.amplitude, self.frequency, self.offset, self.damping): + y += a*np.cos(f*np.pi*(x-o)) * np.exp(-d*(x-o)**2) + return y + + def randomize(self, print_vals=False): + self.amplitude = np.random.uniform(low=self.amp_range[0], high=self.amp_range[1], size=self.n_components) + self.frequency = np.random.uniform(low=self.f_range[0], high=self.f_range[1], size=self.n_components) + self.offset = np.random.uniform(low=self.off_range[0], high=self.off_range[1], size=self.n_components) + self.damping = np.random.uniform(low=self.damp_range[0], high=self.damp_range[1], size=self.n_components) + if print_vals: + self.print_values() + + def print_values(self): + print "a: {0}, f: {1}, o: {2}, d: {3}".format(self.amplitude, self.frequency, self.offset, self.damping) + def damped_wave(x): y = np.cos(6 * np.pi * (x - 0.5)) * np.exp(-10 * (x - 0.5) ** 2) return y From b90ae9da4ab3dd6dc501e823ebd32ee751c3bc4a Mon Sep 17 00:00:00 2001 From: Nicholas Lawrance Date: Thu, 11 May 2017 17:47:58 -0700 Subject: [PATCH 13/38] Changed some active learner stuff, updated plots to match paper notation --- GP_preference_demo.py | 17 ++++++++++------- active_learners.py | 20 ++++++++++++++++++++ active_statruns.py | 24 ++++++------------------ plot_statruns.py | 36 ++++++++++++++++++++++++++---------- plot_tools.py | 8 ++++---- test_data.py | 21 +++++++++++++++++++++ 6 files changed, 87 insertions(+), 39 deletions(-) diff --git a/GP_preference_demo.py b/GP_preference_demo.py index 34688a1..0bd7851 100644 --- a/GP_preference_demo.py +++ b/GP_preference_demo.py @@ -8,17 +8,18 @@ # from scipy.stats import beta plt.rc('font',**{'family':'serif','sans-serif':['Computer Modern Roman']}) plt.rc('text', usetex=True) -np.random.seed(0) +np.random.seed(1) train_hyper = False use_test_data = False verbose = True -#log_hyp = np.log([0.2, 0.5, 0.1, 1.0, 10.0]) # length_scale/s, sigma_f, sigma_n_abs, sigma_beta, v_beta +#log_hyp = np.log([0.2, 0.5, 0.1, 1.0, 10.0]) # length_scale/s, sigma_f, sigma_n_rel, sigma_beta, v_beta # log_hyp = np.log([0.07, 1.0, 0.25, 1.0, 28.1]) # log_hyp = np.log([0.065, 0.8, 0.8, 0.8, 20.0]) # log_hyp = np.log([0.05, 1.5, 0.09, 2.0, 50.0]) -log_hyp = np.log([0.02, 0.6, 0.2, 0.8, 60.0]) +# log_hyp = np.log([0.02, 0.6, 0.2, 0.8, 60.0]) +log_hyp = np.log([0.02, 0.6, 0.1, 1.2, 60.0]) n_rel_train = 100 n_abs_train = 100 @@ -26,7 +27,7 @@ rel_sigma = 0.05 delta_f = 1e-5 -beta_sigma = 0.8 +beta_sigma = 0.5 beta_v = 80.0 n_xplot = 101 @@ -41,8 +42,10 @@ # #random_wave.set_values(a=1.2, f=6.0, o=.2, d=20.0) # random_wave.print_values() -random_wave = test_data.MultiWave(amp_range=[0.6, 1.2], f_range=[10.0, 30.0], off_range=[0.1, 0.9], - damp_range=[250.0, 350.0], n_components=3) +# # random_wave = test_data.MultiWave(amp_range=[0.6, 1.2], f_range=[10.0, 30.0], off_range=[0.1, 0.9], +# damp_range=[250.0, 350.0], n_components=3) +random_wave = test_data.DoubleMultiWave(amp_range=[0.0, 0.4, 0.6, 1.2], f_range=[0.1, 2.0, 10.0, 30.0], off_range=[0.0, 1.0, 0.1, 0.9], + damp_range=[0.0, 50.0, 250.0, 350.0], n_components=3) random_wave.print_values() true_function = random_wave.out @@ -106,7 +109,7 @@ ptt.estimate_plots(x_test, f_true, mu_true, fhat, vhat, E_y, rel_sigma, abs_y_samples, p_abs_y_post, p_rel_y_post, x_abs, y_abs, uv_rel, fuv_rel, y_rel, - t_a=r'Posterior absolute likelihood, $p(y | \mathcal{Y}, \theta)$', + t_a=r'Posterior absolute likelihood, $p(u | \mathcal{Y}, \theta)$', t_r=r'Posterior relative likelihood $P(x_0 \succ x_1 | \mathcal{Y}, \theta)$') plt.show() diff --git a/active_learners.py b/active_learners.py index 44d0d2a..475640e 100644 --- a/active_learners.py +++ b/active_learners.py @@ -111,6 +111,26 @@ def select_observation(self, domain=None, n_test=100, p_rel=0.5, n_rel_samples=2 best_n = [np.argmax(ucb)] # [softmax_selector(ucb, tau=tau)] # return x_test[best_n, :] +class UCBAbsRelD(ActiveLearner): + def select_observation(self, domain=None, n_test=100, p_rel=0.5, n_rel_samples=2, gamma=2.0, tau=5.0): + x_test = self.uniform_domain_sampler(n_test, domain) + fhat, vhat = self.predict_latent(x_test) + ucb = calc_ucb(fhat, vhat, gamma).flatten() + + if np.random.uniform() < p_rel: # i.e choose a relative sample + best_n = [softmax_selector(ucb, tau=tau)] #[np.argmax(ucb)] # + p_rel_y = self.rel_posterior_likelihood_array(fhat=fhat, varhat=vhat) + # sq_dist = GPpref.squared_distance(x_test, x_test) + while len(best_n) < n_rel_samples: + # ucb = ucb*sq_dist[best_n[-1], :] # Discount ucb by distance + ucb[best_n[-1]] = 0.0 + ucb *= 2*p_rel_y[best_n[-1],:]*(1.0 - p_rel_y[best_n[-1],:]) # Divide by likelihood that each point is better than previous best + best_n.append(softmax_selector(ucb, tau=tau*5.0)) + # best_n.append(np.argmax(ucb)) + else: + best_n = [np.argmax(ucb)] # [softmax_selector(ucb, tau=tau)] # + return x_test[best_n, :] + class PeakComparitor(ActiveLearner): def test_observation(self, x, y, x_test, gamma): diff --git a/active_statruns.py b/active_statruns.py index 923988c..7baec08 100644 --- a/active_statruns.py +++ b/active_statruns.py @@ -4,9 +4,10 @@ import matplotlib.pyplot as plt import GPpref import plot_tools as ptt -from active_learners import ActiveLearner, UCBLatent, PeakComparitor, LikelihoodImprovement, ABSThresh, UCBAbsRel +from active_learners import ActiveLearner, UCBLatent, PeakComparitor, LikelihoodImprovement, ABSThresh, UCBAbsRel, UCBAbsRelD import test_data import pickle +import plot_statruns class Learner(object): def __init__(self, model_type, obs_arguments): @@ -24,8 +25,6 @@ def wrms(y_true, y_est, weight=True): return np.sqrt(np.mean(((y_true - y_est)*w)**2)) nowstr = time.strftime("%Y_%m_%d-%H_%M") -plt.rc('font',**{'family':'serif','sans-serif':['Computer Modern Roman']}) -plt.rc('text', usetex=True) # log_hyp = np.log([0.1,0.5,0.1,10.0]) # length_scale, sigma_f, sigma_probit, v_beta # log_hyp = np.log([0.07, 0.75, 0.25, 1.0, 28.1]) @@ -71,12 +70,13 @@ def wrms(y_true, y_est, weight=True): Learner(ActiveLearner, {'p_rel': 1.0, 'n_rel_samples': n_rel_samples}), # 'Random (rel)', Learner(ActiveLearner, {'p_rel': 0.0, 'n_rel_samples': n_rel_samples}), # 'Random (abs)', Learner(UCBLatent, {'gamma': 2.0, 'n_test': 100}), # 'UCBLatent' - Learner(UCBAbsRel, { 'n_test': 100, 'p_rel': 0.5, 'n_rel_samples': n_rel_samples, 'gamma': 2.0, 'tau':5.0}), # 'UCBCombined', + Learner(UCBAbsRel, { 'n_test': 100, 'p_rel': 1.0, 'n_rel_samples': n_rel_samples, 'gamma': 2.0, 'tau':5.0}), # 'UCBCombined', + Learner(UCBAbsRelD, { 'n_test': 100, 'p_rel': 1.0, 'n_rel_samples': n_rel_samples, 'gamma': 2.0, 'tau':5.0}), # Learner(ABSThresh, {'n_test': 100, 'p_thresh': 0.7}), # 'ABSThresh' # Learner(PeakComparitor, {'gamma': 2.0, 'n_test': 50, 'n_rel_samples': n_rel_samples}), # 'PeakComparitor' # Learner(LikelihoodImprovement, {'req_improvement': 0.60, 'n_test': 50, 'gamma': 2.0, 'n_rel_samples': n_rel_samples, 'p_thresh': 0.7}) # 'LikelihoodImprovement' ] -names = ['Random (rel and abs)', 'Random (rel)', 'Random (abs)', 'UCBLatent (abs)', 'UCBCombined (rel and abs)'] +names = ['Random (rel and abs)', 'Random (rel)', 'Random (abs)', 'UCB (abs)', 'UCBC (rel and abs)', 'UCBD (rel and abs)'] n_learners = len(learners) obs_array = [{'name': name, 'obs': []} for name in names] @@ -156,16 +156,4 @@ def wrms(y_true, y_est, weight=True): with open(data_dir+'obs.pkl', 'wb') as fh: pickle.dump(obs_array, fh) -f0, ax0 = plt.subplots() -hl = ax0.plot(np.arange(n_queries+1), np.mean(wrms_results, axis=2).T) -f0.legend(hl, names) - -f1, ax1 = plt.subplots() -hl1 = ax1.plot(np.arange(n_queries+1), np.mean(true_pos_results, axis=2).T) -f1.legend(hl1, names) - -f2, ax2 = plt.subplots() -hl2 = ax2.plot(np.arange(n_queries+1), np.mean(selected_error, axis=2).T) -f2.legend(hl2, names) - -plt.show() \ No newline at end of file +plot_statruns.plot_results(wrms_results, true_pos_results, selected_error, obs_array, data_dir=data_dir, bars=True, norm_comparator=0) \ No newline at end of file diff --git a/plot_statruns.py b/plot_statruns.py index 26ad50f..ff2d7b9 100644 --- a/plot_statruns.py +++ b/plot_statruns.py @@ -4,17 +4,22 @@ from Tkinter import Tk from tkFileDialog import askdirectory # from test_data import ObsObject +plt.rc('font',**{'family':'serif','sans-serif':['Computer Modern Roman']}) +plt.rc('text', usetex=True) - -def single_plot_SEM(data, x=None, names=None, title=None, xlabel='Number of samples', bars=False): +def single_plot(data, x=None, names=None, title=None, xlabel='Number of samples', bars=False, sem=False): mean_err = np.mean(data, axis=2) - std_err_mean = np.std(data, axis=2, ddof=1) + if sem: + err = np.std(data, axis=2, ddof=1)/np.sqrt(data.shape[2]) + else: + err = np.std(data, axis=2) + if x is None: x = np.arange(data.shape[1]) hf, hax = plt.subplots() hl = [] - for mu, sig in zip(mean_err, std_err_mean): + for mu, sig in zip(mean_err, err): if bars: hl.append(hax.errorbar(x, mu, yerr=sig, capsize=2.0)) else: @@ -25,12 +30,22 @@ def single_plot_SEM(data, x=None, names=None, title=None, xlabel='Number of samp return hf, hax -def plot_statrun_results(wrms_results, true_pos_results, selected_error, obs_array, data_dir=None, bars=False): +def plot_results(wrms_results, true_pos_results, selected_error, obs_array, data_dir=None, bars=True, norm_comparator=0): names = [l['name'] for l in obs_array] - f0, ax0 = single_plot_SEM(wrms_results, names=names, title='Weighted RMSE', bars=bars) - f1, ax1 = single_plot_SEM(true_pos_results, names=names, title='True positive selections', bars=False) - f2, ax2 = single_plot_SEM(selected_error, names=names, title='RMSE of best paths', bars=False) + f0, ax0 = single_plot(wrms_results, names=names, title='Weighted RMSE', bars=bars, sem=True) + f1, ax1 = single_plot(true_pos_results, names=names, title='True positive selections', bars=False) + f2, ax2 = single_plot(selected_error, names=names, title='RMSE of best paths', bars=False) + f = [f0, f1, f2] + ax = [ax0, ax1, ax2] + + try: + norm_wrms = wrms_results/wrms_results[norm_comparator] + f3, ax3 = single_plot(norm_wrms, names=names, title='Normalized weighted RMSE', bars=bars, sem=False) + f.append(f3) + ax.append(ax3) + except: + pass if data_dir is not None: f0.savefig(data_dir + '/wrms.pdf', bbox_inches='tight') @@ -40,11 +55,12 @@ def plot_statrun_results(wrms_results, true_pos_results, selected_error, obs_arr # for i in range(mean_err.shape[0]): # hl.append(plt.errorbar(np.arange(mean_err.shape[1]), mean_err[i,:], yerr=std_err[i, :])) # plt.legend(hl, names) + plt.show() return f0, f1, f2 def load_and_plot(save_plots=True, *args, **kwargs): Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing - data_dir = askdirectory() # show an "Open" dialog box and return the path to the selected file + data_dir = askdirectory(initialdir='./data/') # show an "Open" dialog box and return the path to the selected file with open(data_dir+'/wrms.pkl', 'rb') as fh: wrms_results = pickle.load(fh) # Dimensions n_learners, n_queries+1, n_trials @@ -60,7 +76,7 @@ def load_and_plot(save_plots=True, *args, **kwargs): if not save_plots: data_dir = None - plot_statrun_results(wrms_results, true_pos_results, selected_error, obs_array, data_dir=data_dir, *args, **kwargs) + plot_results(wrms_results, true_pos_results, selected_error, obs_array, data_dir=data_dir, *args, **kwargs) plt.show() if __name__ == "__main__": diff --git a/plot_tools.py b/plot_tools.py index a0367cb..fadff11 100644 --- a/plot_tools.py +++ b/plot_tools.py @@ -38,7 +38,7 @@ def plot_setup_rel(t_l = r'Latent function, $f(x)$', t_r = r'Relative likelihood ax_r.set_ylabel('$x_1$') return fig, (ax_l, ax_r) -def plot_setup_2d(t_l = r'Latent function, $f(x)$', t_a = r'Absolute likelihood, $p(y | f(x))$', +def plot_setup_2d(t_l = r'Latent function, $f(x)$', t_a = r'Absolute likelihood, $p(u | f(x))$', t_r = r'Relative likelihood, $P(x_0 \succ x_1 | f(x_0), f(x_1))$'): fig, (ax_l, ax_a, ax_r) = plt.subplots(1, 3) @@ -52,7 +52,7 @@ def plot_setup_2d(t_l = r'Latent function, $f(x)$', t_a = r'Absolute likelihood, # Absolute likelihood ax_a.set_title(t_a) ax_a.set_xlabel('$x$') - ax_a.set_ylabel('$y$') + ax_a.set_ylabel('$u$') # Relative likelihood ax_r.set_title(t_r) @@ -85,7 +85,7 @@ def true_plots(xt, ft, mu_t, rel_sigma, y_samples, p_a_y, p_r_y, xa_train, ya_tr if xa_train.shape[0] > 0: ax_a.plot(xa_train, ya_train, 'w+') h_yt, = ax_a.plot(xt, mu_t, c=lines[0]) - ax_a.legend([h_yt], ['$E(y|f(x))$']) + ax_a.legend([h_yt], ['$E[u]$']) fig.colorbar(h_pat, ax=ax_a) # True relative likelihood @@ -140,7 +140,7 @@ def estimate_plots(xt, ft, mu_t, fhat, vhat, E_y, rel_sigma, if xa_train.shape[0] > 0: ax_a.plot(xa_train, ya_train, 'w+') ax_a.legend([h_yt, hEy], - [r'True mean, $E_{p(y|f(x))}[y]$', r'Posterior mean, $E_{p(y|\mathcal{Y})}\left[y\right]$']) + [r'True mean, $E[u]$', r'Posterior mean, $E_{p(u|\mathcal{Y})}\left[u\right]$']) fig.colorbar(h_pap, ax=ax_a) # Relative posterior likelihood diff --git a/test_data.py b/test_data.py index 09af4b5..cd5a351 100644 --- a/test_data.py +++ b/test_data.py @@ -55,6 +55,27 @@ def randomize(self, print_vals=False): def print_values(self): print "a: {0}, f: {1}, o: {2}, d: {3}".format(self.amplitude, self.frequency, self.offset, self.damping) + +class DoubleMultiWave(object): + def __init__(self, amp_range, f_range, off_range, damp_range, n_components=2): + self.f_low = MultiWave(amp_range[0:2], f_range[0:2], off_range[0:2], damp_range[0:2], n_components=1) + self.f_high = MultiWave(amp_range[2:4], f_range[2:4], off_range[2:4], damp_range[2:4], n_components=n_components-1) + self.randomize() + + def out(self, x): + return self.f_low.out(x)+self.f_high.out(x) + + def randomize(self, print_vals=False): + self.f_low.randomize() + self.f_high.randomize() + if print_vals: + self.print_values() + + def print_values(self): + self.f_low.print_values() + self.f_high.print_values() + + def damped_wave(x): y = np.cos(6 * np.pi * (x - 0.5)) * np.exp(-10 * (x - 0.5) ** 2) return y From db9ea3cf39fd08b5ae61c969800aef0760397897 Mon Sep 17 00:00:00 2001 From: Nicholas Lawrance Date: Tue, 8 Aug 2017 13:37:16 -0700 Subject: [PATCH 14/38] Messed around with active learners, sampling from posterior --- GP_preference_demo.py | 39 ++++---- GPpref.py | 36 +++++-- active_learners.py | 176 ++++++++++++++++++++++++++++++--- active_statruns.py | 186 +++++++++++++++++++++-------------- nice_plot_colors.py | 11 ++- plot_acquisition.py | 135 +++++++++++++++++++++++++ plot_statruns.py | 73 +++++++++----- plot_tools.py | 21 +++- predictive_entropy_search.py | 0 pref_active_learning.py | 23 +++-- test_data.py | 21 ++++ 11 files changed, 570 insertions(+), 151 deletions(-) create mode 100644 plot_acquisition.py create mode 100644 predictive_entropy_search.py diff --git a/GP_preference_demo.py b/GP_preference_demo.py index 0bd7851..b06dcf4 100644 --- a/GP_preference_demo.py +++ b/GP_preference_demo.py @@ -15,48 +15,43 @@ verbose = True #log_hyp = np.log([0.2, 0.5, 0.1, 1.0, 10.0]) # length_scale/s, sigma_f, sigma_n_rel, sigma_beta, v_beta -# log_hyp = np.log([0.07, 1.0, 0.25, 1.0, 28.1]) -# log_hyp = np.log([0.065, 0.8, 0.8, 0.8, 20.0]) -# log_hyp = np.log([0.05, 1.5, 0.09, 2.0, 50.0]) -# log_hyp = np.log([0.02, 0.6, 0.2, 0.8, 60.0]) -log_hyp = np.log([0.02, 0.6, 0.1, 1.2, 60.0]) -n_rel_train = 100 -n_abs_train = 100 +# random_wave = test_data.MultiWave(amp_range=[0.6, 1.2], f_range=[10.0, 30.0], off_range=[0.1, 0.9], damp_range=[250.0, 350.0], n_components=3) +# log_hyp = np.log([0.018, 1.0, 0.1, 0.6, 60.0]) -rel_sigma = 0.05 +random_wave = test_data.MultiWave(amp_range=[1.2, 3.0], f_range=[5.0, 8.0], off_range=[0.1, 0.9], damp_range=[30.0, 50.0], n_components=2) +log_hyp = np.log([0.1, 2.0, 0.1, 5.0, 30.0]) + +n_rel_train = 10 +n_abs_train = 0 + +rel_sigma = 0.1 delta_f = 1e-5 -beta_sigma = 0.5 -beta_v = 80.0 +beta_sigma = 1.5 +beta_v = 40.0 n_xplot = 101 n_mcsamples = 1000 n_ysamples = 101 +n_posterior_samples = 3 # Define polynomial function to be modelled -#true_function = test_data.zero_fun - # random_wave = test_data.VariableWave(amp_range=[0.6, 1.2], f_range=[5.0, 10.0], off_range=[0.2, 0.8], # damp_range=[30.0, 100.0]) -# #random_wave.set_values(a=1.2, f=6.0, o=.2, d=20.0) -# random_wave.print_values() +# random_wave.set_values(a=1.2, f=6.0, o=.2, d=20.0) +# random_wave = test_data.DoubleMultiWave(amp_range=[0.0, 0.4, 0.6, 1.2], f_range=[0.1, 2.0, 10.0, 30.0], off_range=[0.0, 1.0, 0.1, 0.9], damp_range=[0.0, 50.0, 250.0, 350.0], n_components=3) +# true_function = test_data.damped_wave -# # random_wave = test_data.MultiWave(amp_range=[0.6, 1.2], f_range=[10.0, 30.0], off_range=[0.1, 0.9], -# damp_range=[250.0, 350.0], n_components=3) -random_wave = test_data.DoubleMultiWave(amp_range=[0.0, 0.4, 0.6, 1.2], f_range=[0.1, 2.0, 10.0, 30.0], off_range=[0.0, 1.0, 0.1, 0.9], - damp_range=[0.0, 50.0, 250.0, 350.0], n_components=3) random_wave.print_values() true_function = random_wave.out -# true_function = test_data.damped_wave - rel_obs_fun = GPpref.RelObservationSampler(true_function, GPpref.PrefProbit(sigma=rel_sigma)) abs_obs_fun = GPpref.AbsObservationSampler(true_function, GPpref.AbsBoundProbit(sigma=beta_sigma, v=beta_v)) # Main program # True function -x_plot = np.linspace(0.0,1.0,n_xplot,dtype='float') +x_plot = np.linspace(-1.0,1.0,n_xplot,dtype='float') x_test = np.atleast_2d(x_plot).T f_true = abs_obs_fun.f(x_test) mu_true = abs_obs_fun.mean_link(x_test) @@ -108,7 +103,7 @@ fig_p, (ax_p_l, ax_p_a, ax_p_r) = \ ptt.estimate_plots(x_test, f_true, mu_true, fhat, vhat, E_y, rel_sigma, abs_y_samples, p_abs_y_post, p_rel_y_post, - x_abs, y_abs, uv_rel, fuv_rel, y_rel, + x_abs, y_abs, uv_rel, fuv_rel, y_rel, n_posterior_samples=n_posterior_samples, t_a=r'Posterior absolute likelihood, $p(u | \mathcal{Y}, \theta)$', t_r=r'Posterior relative likelihood $P(x_0 \succ x_1 | \mathcal{Y}, \theta)$') diff --git a/GPpref.py b/GPpref.py index d83a111..ed8c4fe 100644 --- a/GPpref.py +++ b/GPpref.py @@ -6,23 +6,26 @@ from scipy.stats import norm, beta from scipy.linalg import block_diag -#define a standard normal pdf +# Define a standard normal pdf _sqrt_2pi = np.sqrt(2*np.pi) def std_norm_pdf(x): x = np.clip(x,-1e150,1e150) return norm.pdf(x) - #return np.exp(-(x**2)/2)/_sqrt_2pi + # return np.exp(-(x**2)/2)/_sqrt_2pi + def std_norm_cdf(x): x = np.clip(x, -30, 100 ) return norm.cdf(x) + def norm_pdf_norm_cdf_ratio(z): # Inverse Mills ratio for stability out = -z out[z>-30] = std_norm_pdf(z[z>-30])/std_norm_cdf(z[z>-30]) return out + # Define squared distance calculation function def squared_distance(A,B): A = np.reshape(A,(len(A),1)) @@ -35,6 +38,7 @@ def squared_distance(A,B): sqDist = A2 + B2 - AB return sqDist + def print_hyperparameters(theta, log=False): if log is True: theta = np.exp(theta) @@ -42,6 +46,7 @@ def print_hyperparameters(theta, log=False): lstr = ', '.join(['%.2f']*nl) % tuple(theta[:nl]) print "l: {0}, sig_f: {1:0.2f}, sig: {2:0.2f}, v: {3:0.2f}".format(lstr, theta[-3], theta[-2], theta[-1]) + # Define squared exponential CovarianceFunction function class SquaredExponential(object): def __init__(self, logHyp, x): @@ -168,16 +173,16 @@ def mean_link(self, f): ml = np.clip(std_norm_cdf(f*self._isqrt2sig), 1e-12, 1.0-1e-12) return ml - def alpha(self, f): - return self.v * self.mean_link(f) - - def beta(self, f): - return self.v * (1-self.mean_link(f)) + # def alpha(self, f): + # return self.v * self.mean_link(f) + # + # def beta(self, f): + # return self.v * (1-self.mean_link(f)) def get_alpha_beta(self, f): ml = self.mean_link(f) aa = self.v * ml - bb = self.v * (1-ml) + bb = self.v - aa # = self.v * (1-ml) return aa, bb def likelihood(self, y, f): @@ -187,6 +192,10 @@ def likelihood(self, y, f): def log_likelihood(self, y, f): return np.log(self.likelihood(y,f)) + def cdf(self, y, f): + aa, bb = self.get_alpha_beta(f) + return beta.cdf(y, aa, bb) + def derivatives(self, y, f): aa, bb = self.get_alpha_beta(f) @@ -391,6 +400,12 @@ def predict_latent(self, x): var_latent = Ktt - np.matmul(kt.T, np.matmul(iKW, np.matmul(self.W, kt))) return mean_latent, var_latent + def sample_latent_posterior(self, mean, covariance, n_samples = 1): + assert mean.shape[0] == covariance.shape[0] + assert covariance.shape[1] == covariance.shape[0] + y_post = np.random.multivariate_normal(mean.flatten(), covariance, n_samples) + return y_post + def _check_latent_input(self, x=None, fhat=None, varhat=None): if (fhat is None or varhat is None): if x is not None: @@ -464,7 +479,10 @@ def _gen_x_obs(n, n_xdim=1, domain=None): x_test = x_test * np.diff(domain, axis=0) + domain[0, :] return x_test - def cheat_multi_sampler(self, x): + def gaussian_multi_pairwise_sampler(self, x): + # Find the maximum from a set of n samples (x should be n by d) + # Return pairwise relationships that one point is higher than the others + # Sample from Gaussian distributions around function values fx = np.random.normal(loc=self.f(x), scale=self.l.sigma) max_xi = np.argmax(fx) other_xi = np.delete(np.arange(x.shape[0]), max_xi) diff --git a/active_learners.py b/active_learners.py index 475640e..3912c2a 100644 --- a/active_learners.py +++ b/active_learners.py @@ -8,6 +8,7 @@ def calc_ucb(fhat, vhat, gamma=2.0, sigma_offset=0.0): return fhat + gamma * (np.sqrt(np.atleast_2d(vhat.diagonal()).T) - sigma_offset) def softmax_selector(x, tau=1.0): + # High tau is more random ex = np.exp((x - x.max())/tau) Px = ex/ex.sum() return np.random.choice(len(x), p=Px) @@ -42,6 +43,14 @@ def uniform_domain_sampler(self, n_samples, domain=None): x_test = x_test*np.diff(domain, axis=0) + domain[0, :] return x_test + def linear_domain_sampler(self, n_samples, domain=None): + # One dimension only at the moment! + assert self._xdim == 1 + x_test = np.atleast_2d(np.linspace(.0, 1.0, n_samples, self._xdim)).T + if domain is not None: + x_test = x_test*np.diff(domain, axis=0) + domain[0, :] + return x_test + def create_posterior_plot(self, x_test, f_true, mu_true, rel_sigma, fuv_train, abs_y_samples, mc_samples): # Latent predictions fhat, vhat = self.predict_latent(x_test) @@ -73,6 +82,14 @@ def select_observation(self, domain=None, n_test=100, gamma=2.0): ucb = calc_ucb(fhat, vhat, gamma) return x_test[[np.argmax(ucb)], :] +class UCBLatentSoftmax(ActiveLearner): + # All absolute returns + def select_observation(self, domain=None, n_test=100, gamma=2.0, tau=1.0): + x_test = self.uniform_domain_sampler(n_test, domain) + fhat, vhat = self.predict_latent(x_test) + ucb = calc_ucb(fhat, vhat, gamma).flatten() + return x_test[[softmax_selector(ucb, tau)], :] + class UCBOut(ActiveLearner): # NOT FULLY IMPLEMENTED - BROKEN def select_observation(self, domain=None, n_test=100, gamma=2.0): @@ -82,6 +99,70 @@ def select_observation(self, domain=None, n_test=100, gamma=2.0): Ey = self.expected_y(x_test, fhat, vhat) return x_test[[np.argmax(Ey)], :] + +class ProbabilityImprovementAbs(ActiveLearner): + def mean_var_sampler(self, n_test, domain): + x_test = self.uniform_domain_sampler(n_test, domain) + fhat, vhat = self.predict_latent(x_test) + shat = np.atleast_2d(np.sqrt(vhat.diagonal())).T + return x_test, fhat, shat + + @staticmethod + def delta(f_star, f_est, sig_est, zeta): + return (f_est - f_star - zeta) / sig_est + + def probability_improvement(self, f_star, f_est, sig_est, zeta): + Z = self.delta(f_star, f_est, sig_est, zeta) + PI = GPpref.std_norm_cdf(Z) + return PI + + def select_observation(self, domain=None, n_test=100, zeta=0.0, *args, **kwargs): + f_star = self.f.max() + x_test, fhat, shat = self.mean_var_sampler(n_test, domain) + PI = self.probability_improvement(f_star, fhat, shat, zeta) + return x_test[[np.argmax(PI)], :] + + +class ProbabilityImprovementRel(ProbabilityImprovementAbs): + def select_observation(self, domain=None, n_test=100, zeta=0.0, *args, **kwargs): + i_star = np.argmax(self.f) + x_out = np.zeros((2, self.x_train_all.shape[1]), dtype='float') + x_out[0] = self.x_train_all[i_star] + x_out[1] = super(ProbabilityImprovementRel, self).select_observation(domain, n_test, zeta, *args, **kwargs)[0] + return x_out + + +class ExpectedImprovementAbs(ProbabilityImprovementAbs): + def expected_improvement(self, f_star, f_est, sig_est, zeta): + Z = self.delta(f_star, f_est, sig_est, zeta) + EI = sig_est * (Z * GPpref.std_norm_cdf(Z) + GPpref.std_norm_pdf(Z)) + return EI + + def select_observation(self, domain=None, n_test=100, zeta=0.0, *args, **kwargs): + f_star = self.f.max() + x_test, fhat, shat = self.mean_var_sampler(n_test, domain) + EI = self.expected_improvement(f_star, fhat, shat, zeta) + return x_test[[np.argmax(EI)], :] + + +class ExpectedImprovementRel(ExpectedImprovementAbs): + def select_observation(self, domain=None, n_test=100, zeta=0.0, *args, **kwargs): + i_star = np.argmax(self.f) + x_out = np.zeros((2, self.x_train_all.shape[1]), dtype='float') + x_out[0] = self.x_train_all[i_star] + x_out[1] = super(ExpectedImprovementRel, self).select_observation(domain, n_test, zeta, *args, **kwargs)[0] + return x_out + + +# class PredictiveEntropy(ActiveLearner): +# # All absolute returns +# def select_observation(self, domain=None, n_test=100): +# +# x_test = self.uniform_domain_sampler(n_test, domain) +# fhat, vhat = self.predict_latent(x_test) +# ucb = calc_ucb(fhat, vhat, gamma).flatten() +# return x_test[[softmax_selector(ucb, tau)], :] + class ABSThresh(ActiveLearner): def select_observation(self, domain=None, n_test=100, p_thresh=0.7): x_test = self.uniform_domain_sampler(n_test, domain) @@ -92,27 +173,28 @@ def select_observation(self, domain=None, n_test=100, p_thresh=0.7): return x_test[[np.argmax(p_under_thresh * (1.0 - p_under_thresh))], :] class UCBAbsRel(ActiveLearner): - def select_observation(self, domain=None, n_test=100, p_rel=0.5, n_rel_samples=2, gamma=2.0, tau=5.0): + def select_observation(self, domain=None, n_test=100, p_rel=0.5, n_rel_samples=2, gamma=2.0, tau=1.0): x_test = self.uniform_domain_sampler(n_test, domain) fhat, vhat = self.predict_latent(x_test) ucb = calc_ucb(fhat, vhat, gamma).flatten() if np.random.uniform() < p_rel: # i.e choose a relative sample best_n = [softmax_selector(ucb, tau=tau)] #[np.argmax(ucb)] # - # p_rel_y = self.rel_posterior_likelihood_array(fhat=fhat, varhat=vhat) - sq_dist = GPpref.squared_distance(x_test, x_test) + # sq_dist = GPpref.squared_distance(x_test, x_test) + p_rel_y = self.rel_posterior_likelihood_array(fhat=fhat, varhat=vhat) + while len(best_n) < n_rel_samples: # ucb = ucb*sq_dist[best_n[-1], :] # Discount ucb by distance ucb[best_n[-1]] = 0.0 - # ucb /= p_rel_y[best_n[-1],:] # Divide by likelihood that each point is better than previous best - best_n.append(softmax_selector(ucb, tau=tau*5.0)) + ucb *= 2*p_rel_y[best_n[-1],:]*(1.0 - p_rel_y[best_n[-1],:]) # Divide by likelihood that each point is better than previous best + best_n.append(softmax_selector(ucb, tau=tau)) # best_n.append(np.argmax(ucb)) else: - best_n = [np.argmax(ucb)] # [softmax_selector(ucb, tau=tau)] # + best_n = [softmax_selector(ucb, tau=tau/2.0)] #[np.argmax(ucb)] # return x_test[best_n, :] class UCBAbsRelD(ActiveLearner): - def select_observation(self, domain=None, n_test=100, p_rel=0.5, n_rel_samples=2, gamma=2.0, tau=5.0): + def select_observation(self, domain=None, n_test=100, p_rel=0.5, n_rel_samples=2, gamma=2.0, tau=1.0): x_test = self.uniform_domain_sampler(n_test, domain) fhat, vhat = self.predict_latent(x_test) ucb = calc_ucb(fhat, vhat, gamma).flatten() @@ -120,17 +202,18 @@ def select_observation(self, domain=None, n_test=100, p_rel=0.5, n_rel_samples=2 if np.random.uniform() < p_rel: # i.e choose a relative sample best_n = [softmax_selector(ucb, tau=tau)] #[np.argmax(ucb)] # p_rel_y = self.rel_posterior_likelihood_array(fhat=fhat, varhat=vhat) - # sq_dist = GPpref.squared_distance(x_test, x_test) + # sq_dist = np.sqrt(GPpref.squared_distance(x_test, x_test)) while len(best_n) < n_rel_samples: - # ucb = ucb*sq_dist[best_n[-1], :] # Discount ucb by distance + # ucb *= sq_dist[best_n[-1], :] # Discount ucb by distance ucb[best_n[-1]] = 0.0 ucb *= 2*p_rel_y[best_n[-1],:]*(1.0 - p_rel_y[best_n[-1],:]) # Divide by likelihood that each point is better than previous best - best_n.append(softmax_selector(ucb, tau=tau*5.0)) - # best_n.append(np.argmax(ucb)) + best_n.append(softmax_selector(ucb, tau=tau)) + #best_n.append(np.argmax(ucb)) else: - best_n = [np.argmax(ucb)] # [softmax_selector(ucb, tau=tau)] # + best_n = [softmax_selector(ucb, tau=tau/2.0)] #[np.argmax(ucb)] # return x_test[best_n, :] + class PeakComparitor(ActiveLearner): def test_observation(self, x, y, x_test, gamma): @@ -241,3 +324,72 @@ def select_observation(self, domain=None, n_test=50, req_improvement=0.6, n_rel_ return x_test[xi, :] +class SampledThreshold(PeakComparitor): + # TODO: FINISH IMPLEMENTATION + + def calculate_threshold_utility(self, fhat, vhat, n_samples, y_threshold): + y_post = self.sample_latent_posterior(fhat, vhat, n_samples = n_samples) + threshold_utility = 0.0 + # For each one, evaluate the probability mass above the threshold in the absolute function + for yi in y_post: + threshold_utility += self.mean_P_above_threshold(yi, y_threshold) + return threshold_utility/n_samples + + def mean_P_above_threshold(self, f, y_threshold): + pmass = 0.0 + for fi in f: + pmass += 1.0 - self.abs_likelihood.cdf(y_threshold, fi) + return pmass/f.shape[0] + + def test_observation(self, x, y, x_test, n_samples, y_threshold): + self.store_observations() + self.add_observations(x, y, self._default_uvi) + util, f, v = self.calculate_threshold_utility(x_test, n_samples, y_threshold) + self.reset_observations() + return util + + def select_observation(self, domain=None, x_test=None, n_test=100, n_samples=100, y_threshold=0.8): + # Generate a set of test points in the domain (if not specified) + if x_test is None: + x_test = self.linear_domain_sampler(n_test, domain) + + # Sample a set of functions from the current posterior + self.solve_laplace() + fhat, vhat = self.predict_latent(x_test) + base_utility = self.calculate_threshold_utility(fhat, vhat, n_samples, y_threshold) + max_xi = np.argmax(fhat) + other_xi = np.delete(np.arange(n_test), max_xi) + uvi = np.vstack((max_xi * np.ones(n_test - 1, dtype='int'), other_xi)).T + p_pref = self.rel_likelihood.posterior_likelihood(fhat, vhat, uvi, y=-1) + V = np.zeros(n_test - 1) + x = np.zeros((2, 1), dtype='float') + x[0] = x_test[max_xi] + + # Now calculate the expected value for each observation pair + for i,uvi1 in enumerate(other_xi): + x[1] = x_test[uvi1] + V[i] += p_pref[i]*self.test_observation(x, self._minus_y_obs, x_test, max_xi) + if (1-p_pref[i]) > 1e-3: + V[i] += (1-p_pref[i])*self.test_observation(x, self._plus_y_obs, x_test, max_xi) + + Vmax = V.max() + # best_n = np.argpartition(V, -n_comparators)[-n_comparators:] + # best = np.argmax(V) + print 'V_max = {0}'.format(Vmax) + + if Vmax < req_improvement: + # aa, bb = self.abs_likelihood.get_alpha_beta(fhat) + # p_under_thresh = beta.cdf(p_thresh, aa, bb) + # return x_test[[np.argmax(p_under_thresh*(1.0-p_under_thresh))], :] + ucb = calc_ucb(fhat, vhat, gamma, self.rel_likelihood.sigma) + return x_test[[np.argmax(ucb)], :] + else: + best_n = [] + while len(best_n) < n_comparators: + cbest = np.argmax(V) + best_n.append(cbest) + V = V * np.sqrt(GPpref.squared_distance(x_test[[other_xi[cbest]], :], x_test[other_xi])[0]) + xi = np.zeros(n_comparators+1, dtype='int') + xi[0] = max_xi + xi[1:] = other_xi[best_n] + return x_test[xi, :] \ No newline at end of file diff --git a/active_statruns.py b/active_statruns.py index 7baec08..34f0033 100644 --- a/active_statruns.py +++ b/active_statruns.py @@ -1,10 +1,10 @@ # Simple 1D GP classification example import time import numpy as np -import matplotlib.pyplot as plt import GPpref import plot_tools as ptt -from active_learners import ActiveLearner, UCBLatent, PeakComparitor, LikelihoodImprovement, ABSThresh, UCBAbsRel, UCBAbsRelD +import active_learners +# from active_learners import ActiveLearner, UCBLatent, UCBLatentSoftmax, LikelihoodImprovement, ABSThresh, UCBAbsRel, UCBAbsRelD import test_data import pickle import plot_statruns @@ -17,6 +17,7 @@ def __init__(self, model_type, obs_arguments): def build_model(self, training_data): self.model = self.model_type(**training_data) + def wrms(y_true, y_est, weight=True): if weight: w = y_true @@ -24,59 +25,69 @@ def wrms(y_true, y_est, weight=True): w = 1.0 return np.sqrt(np.mean(((y_true - y_est)*w)**2)) -nowstr = time.strftime("%Y_%m_%d-%H_%M") + +def wrms2(y_true, y_est): + w = np.power(np.maximum(y_true, y_est), 2) + return np.sqrt(np.mean(((y_true - y_est)*w)**2)) + + +now_time = time.strftime("%Y_%m_%d-%H_%M") # log_hyp = np.log([0.1,0.5,0.1,10.0]) # length_scale, sigma_f, sigma_probit, v_beta # log_hyp = np.log([0.07, 0.75, 0.25, 1.0, 28.1]) # log_hyp = np.log([0.05, 1.5, 0.09, 2.0, 50.0]) -log_hyp = np.log([0.02, 0.6, 0.2, 0.8, 60.0]) -np.random.seed(10) +log_hyp = np.log([0.018, 1.0, 0.2, 0.5, 60.0]) +np.random.seed(1) n_rel_train = 1 n_abs_train = 0 rel_sigma = 0.05 delta_f = 1e-5 -beta_sigma = 0.8 +beta_sigma = 0.5 beta_v = 80.0 n_xtest = 101 n_best_points = 15 -n_mcsamples = 1000 -n_ysamples = 101 +# n_ysamples = 101 n_trials = 100 +randomize_waves = True n_rel_samples = 5 -n_queries = 20 +n_queries = 80 # Define polynomial function to be modelled # random_wave = test_data.VariableWave([0.6, 1.0], [5.0, 10.0], [0.0, 1.0], [10.0, 20.0]) random_wave = test_data.MultiWave(amp_range=[0.6, 1.2], f_range=[10.0, 30.0], off_range=[0.1, 0.9], damp_range=[250.0, 350.0], n_components=3) -nowstr = time.strftime("%Y_%m_%d-%H_%M") -data_dir = 'data/' + nowstr + '/' +now_time = time.strftime("%Y_%m_%d-%H_%M") +data_dir = 'data/' + now_time + '/' ptt.ensure_dir(data_dir) print "Data will be saved to: {0}".format(data_dir) - +waver = test_data.WaveSaver(n_trials, random_wave.n_components) # True function -x_plot = np.linspace(0.0,1.0,n_xtest,dtype='float') +x_plot = np.linspace(0.0, 1.0, n_xtest,dtype='float') x_test = np.atleast_2d(x_plot).T # Construct active learner object -learners = [Learner(ActiveLearner, {'p_rel': 0.5, 'n_rel_samples': n_rel_samples}), # 'Random (rel and abs)', - Learner(ActiveLearner, {'p_rel': 1.0, 'n_rel_samples': n_rel_samples}), # 'Random (rel)', - Learner(ActiveLearner, {'p_rel': 0.0, 'n_rel_samples': n_rel_samples}), # 'Random (abs)', - Learner(UCBLatent, {'gamma': 2.0, 'n_test': 100}), # 'UCBLatent' - Learner(UCBAbsRel, { 'n_test': 100, 'p_rel': 1.0, 'n_rel_samples': n_rel_samples, 'gamma': 2.0, 'tau':5.0}), # 'UCBCombined', - Learner(UCBAbsRelD, { 'n_test': 100, 'p_rel': 1.0, 'n_rel_samples': n_rel_samples, 'gamma': 2.0, 'tau':5.0}), +learners = [Learner(active_learners.ActiveLearner, {'p_rel': 0.5, 'n_rel_samples': n_rel_samples}), # 'Random (rel and abs)', + Learner(active_learners.ActiveLearner, {'p_rel': 1.0, 'n_rel_samples': n_rel_samples}), # 'Random (rel)', + Learner(active_learners.ActiveLearner, {'p_rel': 0.0, 'n_rel_samples': n_rel_samples}), # 'Random (abs)', + Learner(active_learners.UCBLatent, {'gamma': 4.0, 'n_test': 100}), # 'UCBLatent' + # Learner(UCBLatentSoftmax, {'gamma': 2.0, 'n_test': 100, 'tau':1.0}), # 'UCBSoft (abs)', + Learner(active_learners.ExpectedImprovementAbs, { 'n_test': 100 }), # 'EI (Abs)', + Learner(active_learners.ExpectedImprovementRel, { 'n_test': 100 }), # 'EI (Rel)', + Learner(active_learners.UCBAbsRel, { 'n_test': 100, 'n_rel_samples': n_rel_samples, 'gamma': 2.0, 'tau': 1.0}), # 'UCBCombined', + # Learner(UCBAbsRelD, { 'n_test': 100, 'n_rel_samples': n_rel_samples, 'gamma': 2.0, 'tau': 1.0}), # , 'UCBD (rel and abs)' # Learner(ABSThresh, {'n_test': 100, 'p_thresh': 0.7}), # 'ABSThresh' # Learner(PeakComparitor, {'gamma': 2.0, 'n_test': 50, 'n_rel_samples': n_rel_samples}), # 'PeakComparitor' # Learner(LikelihoodImprovement, {'req_improvement': 0.60, 'n_test': 50, 'gamma': 2.0, 'n_rel_samples': n_rel_samples, 'p_thresh': 0.7}) # 'LikelihoodImprovement' ] -names = ['Random (rel and abs)', 'Random (rel)', 'Random (abs)', 'UCB (abs)', 'UCBC (rel and abs)', 'UCBD (rel and abs)'] +names = ['Random (rel and abs)','Random (rel only)', 'Random (abs only)', 'UCB (abs only)', 'EI (Abs)', 'EI (Rel)', 'UCBCombined (rel and abs)'] +assert len(names) == len(learners), "Number of names does not match number of learners." n_learners = len(learners) obs_array = [{'name': name, 'obs': []} for name in names] @@ -85,64 +96,91 @@ def wrms(y_true, y_est, weight=True): true_pos_results = np.zeros((n_learners, n_queries+1, n_trials), dtype='int') selected_error = np.zeros((n_learners, n_queries+1, n_trials)) -for trial_number in range(n_trials): - print 'Trial {0}'.format(trial_number) - random_wave.randomize(print_vals=True) - rel_obs_fun = GPpref.RelObservationSampler(random_wave.out, GPpref.PrefProbit(sigma=rel_sigma)) - abs_obs_fun = GPpref.AbsObservationSampler(random_wave.out, GPpref.AbsBoundProbit(sigma=beta_sigma, v=beta_v)) - f_true = abs_obs_fun.f(x_test) - y_abs_true = abs_obs_fun.mean_link(x_test) - best_points = np.argpartition(y_abs_true.flatten(), -n_best_points)[-n_best_points:] - best_points_set = set(best_points) - abs_y_samples = np.atleast_2d(np.linspace(0.01, 0.99, n_ysamples)).T - p_abs_y_true = abs_obs_fun.observation_likelihood_array(x_test, abs_y_samples) - p_rel_y_true = rel_obs_fun.observation_likelihood_array(x_test) - - # Initial data - x_rel, uvi_rel, uv_rel, y_rel, fuv_rel = rel_obs_fun.generate_n_observations(n_rel_train, n_xdim=1) - x_abs, y_abs, mu_abs = abs_obs_fun.generate_n_observations(n_abs_train, n_xdim=1) - training_data = {'x_rel': x_rel, 'uvi_rel': uvi_rel, 'x_abs': x_abs, 'y_rel': y_rel, 'y_abs': y_abs, - 'delta_f': delta_f, 'rel_likelihood': GPpref.PrefProbit(), - 'abs_likelihood': GPpref.AbsBoundProbit()} - - # Get initial solution - for nl, learner in enumerate(learners): - learner.build_model(training_data) - learner.model.set_hyperparameters(log_hyp) - f = learner.model.solve_laplace() - fhat, vhat = learner.model.predict_latent(x_test) - y_abs_est = learner.model.abs_posterior_mean(x_test, fhat, vhat) - wrms_results[nl, 0, trial_number] = wrms(y_abs_true, y_abs_est) - - for obs_num in range(n_queries): - learners[4].obs_arguments['p_rel'] = max(0.0, (20-obs_num)/20.0) +trial_number = 0 +# for trial_number in range(n_trials): +while trial_number < n_trials: + + try: + print 'Trial {0}'.format(trial_number) + if randomize_waves: + random_wave.randomize() + random_wave.print_values() + waver.set_vals(trial_number, *random_wave.get_values()) + rel_obs_fun = GPpref.RelObservationSampler(random_wave.out, GPpref.PrefProbit(sigma=rel_sigma)) + abs_obs_fun = GPpref.AbsObservationSampler(random_wave.out, GPpref.AbsBoundProbit(sigma=beta_sigma, v=beta_v)) + f_true = abs_obs_fun.f(x_test) + y_abs_true = abs_obs_fun.mean_link(x_test) + best_points = np.argpartition(y_abs_true.flatten(), -n_best_points)[-n_best_points:] + best_points_set = set(best_points) + # abs_y_samples = np.atleast_2d(np.linspace(0.01, 0.99, n_ysamples)).T + # p_abs_y_true = abs_obs_fun.observation_likelihood_array(x_test, abs_y_samples) + # p_rel_y_true = rel_obs_fun.observation_likelihood_array(x_test) + + # Initial data + x_rel, uvi_rel, uv_rel, y_rel, fuv_rel = rel_obs_fun.generate_n_observations(n_rel_train, n_xdim=1) + x_abs, y_abs, mu_abs = abs_obs_fun.generate_n_observations(n_abs_train, n_xdim=1) + training_data = {'x_rel': x_rel, 'uvi_rel': uvi_rel, 'x_abs': x_abs, 'y_rel': y_rel, 'y_abs': y_abs, + 'delta_f': delta_f, 'rel_likelihood': GPpref.PrefProbit(), + 'abs_likelihood': GPpref.AbsBoundProbit()} + + # Get initial solution for nl, learner in enumerate(learners): - next_x = learner.model.select_observation(**learner.obs_arguments) - if next_x.shape[0] == 1: - next_y, next_f = abs_obs_fun.generate_observations(next_x) - learner.model.add_observations(next_x, next_y) - # print 'Abs: x:{0}, y:{1}'.format(next_x[0], next_y[0]) - else: - next_y, next_uvi, next_fx = rel_obs_fun.cheat_multi_sampler(next_x) - next_fuv = next_fx[next_uvi][:,:,0] - fuv_rel = np.concatenate((fuv_rel, next_fuv), 0) - learner.model.add_observations(next_x, next_y, next_uvi) - # print 'Rel: x:{0}, best_index:{1}'.format(next_x.flatten(), next_uvi[0, 1]) + learner.build_model(training_data) + learner.model.set_hyperparameters(log_hyp) f = learner.model.solve_laplace() fhat, vhat = learner.model.predict_latent(x_test) y_abs_est = learner.model.abs_posterior_mean(x_test, fhat, vhat) best_points_est = set(np.argpartition(y_abs_est.flatten(), -n_best_points)[-n_best_points:]) - true_pos_results[nl, obs_num+1, trial_number] = len(best_points_set.intersection(best_points_est)) - wrms_results[nl, obs_num+1, trial_number] = wrms(y_abs_true, y_abs_est) - selected_error[nl, obs_num + 1, trial_number] = wrms(y_abs_true[best_points], y_abs_est[best_points], weight=False) - - - print true_pos_results[:, obs_num+1, trial_number] - print wrms_results[:, obs_num+1, trial_number] - for nl, learner in enumerate(learners): - obs_tuple = learner.model.get_observations() - obs_array[nl]['obs'].append(test_data.ObsObject(*obs_tuple)) + wrms_results[nl, 0, trial_number] = wrms(y_abs_true, y_abs_est) + true_pos_results[nl, 0, trial_number] = len(best_points_set.intersection(best_points_est)) + selected_error[nl, 0, trial_number] = wrms(y_abs_true[best_points], y_abs_est[best_points], weight=False) + + for obs_num in range(n_queries): + # learners[-2].obs_arguments['p_rel'] = max(0.0, (n_queries-obs_num)/float(n_queries)) + learners[-1].obs_arguments['p_rel'] = max(0.0, (n_queries-obs_num)/float(n_queries)) + for nl, learner in enumerate(learners): + next_x = learner.model.select_observation(**learner.obs_arguments) + if next_x.shape[0] == 1: + next_y, next_f = abs_obs_fun.generate_observations(next_x) + learner.model.add_observations(next_x, next_y) + # print 'Abs: x:{0}, y:{1}'.format(next_x[0], next_y[0]) + else: + next_y, next_uvi, next_fx = rel_obs_fun.gaussian_multi_pairwise_sampler(next_x) + next_fuv = next_fx[next_uvi][:,:,0] + fuv_rel = np.concatenate((fuv_rel, next_fuv), 0) + learner.model.add_observations(next_x, next_y, next_uvi) + # print 'Rel: x:{0}, best_index:{1}'.format(next_x.flatten(), next_uvi[0, 1]) + f = learner.model.solve_laplace() + fhat, vhat = learner.model.predict_latent(x_test) + y_abs_est = learner.model.abs_posterior_mean(x_test, fhat, vhat) + + # Get selected best point set and error results + best_points_est = set(np.argpartition(y_abs_est.flatten(), -n_best_points)[-n_best_points:]) + wrms_results[nl, obs_num+1, trial_number] = wrms2(y_abs_true, y_abs_est) + true_pos_results[nl, obs_num+1, trial_number] = len(best_points_set.intersection(best_points_est)) + selected_error[nl, obs_num+1, trial_number] = wrms(y_abs_true[best_points], y_abs_est[best_points], weight=False) + + + print true_pos_results[:, obs_num+1, trial_number] + print wrms_results[:, obs_num+1, trial_number] + for nl, learner in enumerate(learners): + obs_tuple = learner.model.get_observations() + obs_array[nl]['obs'].append(test_data.ObsObject(*obs_tuple)) + trial_number += 1 + + except RuntimeError: + if randomize_waves == True: + print "Caught a bad laplace, try new trial wave" + continue + else: + raise + except np.linalg.LinAlgError: + if randomize_waves == True: + print "Caught a bad linalg inversion, try new trial wave" + continue + else: + raise with open(data_dir+'wrms.pkl', 'wb') as fh: pickle.dump(wrms_results, fh) @@ -156,4 +194,6 @@ def wrms(y_true, y_est, weight=True): with open(data_dir+'obs.pkl', 'wb') as fh: pickle.dump(obs_array, fh) -plot_statruns.plot_results(wrms_results, true_pos_results, selected_error, obs_array, data_dir=data_dir, bars=True, norm_comparator=0) \ No newline at end of file +waver.save(data_dir+'wave_data.pkl') + +hfig = plot_statruns.plot_results(wrms_results, true_pos_results, selected_error, obs_array, data_dir=data_dir, bars=True, norm_comparator=0) \ No newline at end of file diff --git a/nice_plot_colors.py b/nice_plot_colors.py index 30e946a..c318b28 100644 --- a/nice_plot_colors.py +++ b/nice_plot_colors.py @@ -10,13 +10,22 @@ lines = np.hstack((lines, np.ones((lines.shape[0],1)))) bars = np.hstack((bars, np.ones((bars.shape[0],1)))) + def darken(c,power=2): co = np.array(c).copy() co = np.clip(co**power, 0, 1.0) co[-1] = c[-1] return co + def lighten(c, power=2): co = 1.0-np.array(c).copy() co = darken(co, power) - return 1.0-co \ No newline at end of file + return 1.0-co + + +def greyify(c, grey=0.5, target_level=0.5): + co = np.array(c).copy() + co = np.clip(co + grey*(target_level-co), 0, 1.0) + co[-1] = c[-1] + return co diff --git a/plot_acquisition.py b/plot_acquisition.py new file mode 100644 index 0000000..e828957 --- /dev/null +++ b/plot_acquisition.py @@ -0,0 +1,135 @@ +# Simple 1D GP classification example +import time +import numpy as np +import matplotlib.pyplot as plt +import GPpref +import plot_tools as ptt +import active_learners +import test_data +from matplotlib.backends.backend_pdf import PdfPages + +nowstr = time.strftime("%Y_%m_%d-%H_%M") +plt.rc('font',**{'family':'serif','sans-serif':['Computer Modern Roman']}) +plt.rc('text', usetex=True) + +save_plots = False + +# log_hyp = np.log([0.1,0.5,0.1,10.0]) # length_scale/s, sigma_f, sigma_n_rel, sigma_beta, v_beta +# log_hyp = np.log([0.07, 0.75, 0.25, 1.0, 28.1]) +# log_hyp = np.log([0.05, 1.5, 0.09, 2.0, 50.0]) +log_hyp = np.log([0.02, 0.5, 0.1, 0.8, 60.0]) +np.random.seed(2) + +n_rel_train = 1 +n_abs_train = 1 +rel_sigma = 0.05 +delta_f = 1e-5 + +beta_sigma = 0.8 +beta_v = 80.0 + +n_xplot = 101 +n_mcsamples = 1000 +n_ysamples = 101 + +n_queries = 10 + +# Define polynomial function to be modelled +# true_function = test_data.multi_peak +# random_wave = test_data.VariableWave([0.6, 1.0], [5.0, 10.0], [0.0, 1.0], [10.0, 20.0]) +random_wave = test_data.MultiWave(amp_range=[0.6, 1.2], f_range=[10.0, 30.0], off_range=[0.1, 0.9], + damp_range=[250.0, 350.0], n_components=3) +random_wave.randomize() +# random_wave.set_values(1.0, 6.00, 0.2, 10.50) +true_function = random_wave.out +random_wave.print_values() + +if save_plots: + nowstr = time.strftime("%Y_%m_%d-%H_%M") + fig_dir = 'fig/' + nowstr + '/' + ptt.ensure_dir(fig_dir) + print "Figures will be saved to: {0}".format(fig_dir) + pdf_pages = PdfPages(fig_dir+'posterior_all.pdf') + acq_pdf = PdfPages(fig_dir+'acquisition.pdf') + +rel_obs_fun = GPpref.RelObservationSampler(true_function, GPpref.PrefProbit(sigma=rel_sigma)) +abs_obs_fun = GPpref.AbsObservationSampler(true_function, GPpref.AbsBoundProbit(sigma=beta_sigma, v=beta_v)) + +# True function +x_plot = np.linspace(0.0, 1.0, n_xplot,dtype='float') +x_test = np.atleast_2d(x_plot).T +f_true = abs_obs_fun.f(x_test) +mu_true = abs_obs_fun.mean_link(x_test) +mc_samples = np.random.normal(size=n_mcsamples) +abs_y_samples = np.atleast_2d(np.linspace(0.01, 0.99, n_ysamples)).T +p_abs_y_true = abs_obs_fun.observation_likelihood_array(x_test, abs_y_samples) +p_rel_y_true = rel_obs_fun.observation_likelihood_array(x_test) + +# Training data +x_rel, uvi_rel, uv_rel, y_rel, fuv_rel = rel_obs_fun.generate_n_observations(n_rel_train, n_xdim=1) +x_abs, y_abs, mu_abs = abs_obs_fun.generate_n_observations(n_abs_train, n_xdim=1) + + +# Plot true functions +fig_t, (ax_t_l, ax_t_a, ax_t_r) = ptt.true_plots(x_test, f_true, mu_true, rel_sigma, + abs_y_samples, p_abs_y_true, p_rel_y_true, + x_abs, y_abs, uv_rel, fuv_rel, y_rel, + t_l=r'True latent function, $f(x)$') +if save_plots: + fig_t.savefig(fig_dir+'true.pdf', bbox_inches='tight') + +# Construct active learner object +# learner = active_learners.UCBAbsRelD(x_rel, uvi_rel, x_abs, y_rel, y_abs, delta_f=delta_f, +# rel_likelihood=GPpref.PrefProbit(), abs_likelihood=GPpref.AbsBoundProbit()) +# # obs_arguments = {'req_improvement': 0.60, 'n_test': 50, 'gamma': 2.0, 'n_rel_samples': 5, 'p_thresh': 0.7} +# obs_arguments = {'n_test': 100, 'p_rel': 0.5, 'n_rel_samples': 5, 'gamma': 2.0} + +learner = active_learners.ExpectedImprovementRel(x_rel, uvi_rel, x_abs, y_rel, y_abs, delta_f=delta_f, + rel_likelihood=GPpref.PrefProbit(), abs_likelihood=GPpref.AbsBoundProbit()) +acq_fun = learner.expected_improvement +obs_arguments = {'n_test': 100, 'zeta': 0.1} + +# Get initial solution +learner.set_hyperparameters(log_hyp) +f = learner.solve_laplace() +learner.print_hyperparameters() + +if save_plots: + fig_p, (ax_p_l, ax_p_a, ax_p_r) = learner.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, + abs_y_samples, mc_samples) + fig_a, ax_a = plt.subplots() + pdf_pages.savefig(fig_p, bbox_inches='tight') + # fig_p.savefig(fig_dir+'posterior00.pdf', bbox_inches='tight') + +for obs_num in range(n_queries): + obs_arguments['p_rel'] = max(0.0, (n_queries - obs_num) / float(n_queries)) + next_x = learner.select_observation(**obs_arguments) + y_acq = learner.expected_improvement() + if next_x.shape[0] == 1: + next_y, next_f = abs_obs_fun.generate_observations(next_x) + learner.add_observations(next_x, next_y) + print 'Abs: x:{0}, y:{1}'.format(next_x[0], next_y[0]) + else: + next_y, next_uvi, next_fx = rel_obs_fun.gaussian_multi_pairwise_sampler(next_x) + next_fuv = next_fx[next_uvi][:, :, 0] + fuv_rel = np.concatenate((fuv_rel, next_fuv), 0) + learner.add_observations(next_x, next_y, next_uvi) + print 'Rel: x:{0}, best_index:{1}'.format(next_x.flatten(), next_uvi[0, 1]) + + f = learner.solve_laplace() + if save_plots: + fig_p, (ax_p_l, ax_p_a, ax_p_r) = learner.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, + abs_y_samples, mc_samples) + pdf_pages.savefig(fig_p, bbox_inches='tight') + # fig_p.savefig(fig_dir+'posterior{0:02d}.pdf'.format(obs_num+1), bbox_inches='tight') + plt.close(fig_p) + +learner.print_hyperparameters() + +if not save_plots: + fig_p, (ax_p_l, ax_p_a, ax_p_r) = learner.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, + abs_y_samples, mc_samples) +else: + pdf_pages.close() + +plt.show() diff --git a/plot_statruns.py b/plot_statruns.py index ff2d7b9..f04ef01 100644 --- a/plot_statruns.py +++ b/plot_statruns.py @@ -7,41 +7,62 @@ plt.rc('font',**{'family':'serif','sans-serif':['Computer Modern Roman']}) plt.rc('text', usetex=True) -def single_plot(data, x=None, names=None, title=None, xlabel='Number of samples', bars=False, sem=False): - mean_err = np.mean(data, axis=2) - if sem: - err = np.std(data, axis=2, ddof=1)/np.sqrt(data.shape[2]) - else: - err = np.std(data, axis=2) + +def single_plot(data, x=None, names=None, title='', xlabel='Number of samples', ylabel='', bars=False, percentile=0, precut=0, errorevery=5): + data = data[:,precut:,:] + n_trials = data.shape[2] if x is None: - x = np.arange(data.shape[1]) + x = np.arange(data.shape[1])+precut hf, hax = plt.subplots() hl = [] - for mu, sig in zip(mean_err, err): + for dd in data: + mean_err = np.mean(dd, axis=1) + if percentile < 0: + err_lo = np.std(dd, axis=1, ddof=1) / np.sqrt(n_trials) + err_hi = err_lo + elif percentile == 0: + err_lo = np.std(dd, axis=1) + err_hi = err_lo + else: + err_lo = mean_err - np.percentile(dd, percentile, axis=1) + err_hi = np.percentile(dd, percentile + 50, axis=1) - mean_err + err = np.array([err_lo, err_hi]) + if bars: - hl.append(hax.errorbar(x, mu, yerr=sig, capsize=2.0)) + hl.append(hax.errorbar(x, mean_err, yerr=err, capsize=2.0, errorevery=errorevery)) else: - hl.append(hax.plot(x, mu)[0]) + hl.append(hax.plot(x, mean_err)[0]) hax.legend(hl, names, loc='best') hax.set_title(title) hax.set_xlabel(xlabel) + hax.set_ylabel(ylabel) return hf, hax -def plot_results(wrms_results, true_pos_results, selected_error, obs_array, data_dir=None, bars=True, norm_comparator=0): - names = [l['name'] for l in obs_array] +def plot_results(wrms_results, true_pos_results, selected_error, obs_array, data_dir=None, bars=True, norm_comparator=0, exclusions=[4]): + methods_indexes = [] + for i in range(wrms_results.shape[0]): + if i not in exclusions: + methods_indexes.append(i) + methods_indexes = np.array(methods_indexes) + + names = [obs_array[i]['name'] for i in methods_indexes] - f0, ax0 = single_plot(wrms_results, names=names, title='Weighted RMSE', bars=bars, sem=True) - f1, ax1 = single_plot(true_pos_results, names=names, title='True positive selections', bars=False) - f2, ax2 = single_plot(selected_error, names=names, title='RMSE of best paths', bars=False) + wrms_results=wrms_results[methods_indexes,:,:] + true_pos_results=true_pos_results[methods_indexes,:,:] + selected_error=selected_error[methods_indexes,:,:] + + f0, ax0 = single_plot(wrms_results, names=names, ylabel='Weighted RMSE', bars=bars) + f1, ax1 = single_plot(true_pos_results, names=names, ylabel='True positive selections (out of 15)', bars=True, precut=1, percentile=0) + f2, ax2 = single_plot(selected_error, names=names, ylabel='RMSE of best paths', bars=True, precut=1) f = [f0, f1, f2] ax = [ax0, ax1, ax2] try: norm_wrms = wrms_results/wrms_results[norm_comparator] - f3, ax3 = single_plot(norm_wrms, names=names, title='Normalized weighted RMSE', bars=bars, sem=False) + f3, ax3 = single_plot(norm_wrms, names=names, title='Normalized weighted RMSE', bars=bars, percentile=0) f.append(f3) ax.append(ax3) except: @@ -56,12 +77,9 @@ def plot_results(wrms_results, true_pos_results, selected_error, obs_array, data # hl.append(plt.errorbar(np.arange(mean_err.shape[1]), mean_err[i,:], yerr=std_err[i, :])) # plt.legend(hl, names) plt.show() - return f0, f1, f2 - -def load_and_plot(save_plots=True, *args, **kwargs): - Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing - data_dir = askdirectory(initialdir='./data/') # show an "Open" dialog box and return the path to the selected file + return f +def load_data(data_dir): with open(data_dir+'/wrms.pkl', 'rb') as fh: wrms_results = pickle.load(fh) # Dimensions n_learners, n_queries+1, n_trials @@ -74,10 +92,19 @@ def load_and_plot(save_plots=True, *args, **kwargs): with open(data_dir+'/obs.pkl', 'rb') as fh: obs_array = pickle.load(fh) + return wrms_results, true_pos_results, selected_error, obs_array + +def load_and_plot(save_plots=True, *args, **kwargs): + Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing + data_dir = askdirectory(initialdir='./data/') # open folder GUI + + wrms_results, true_pos_results, selected_error, obs_array = load_data(data_dir) + if not save_plots: data_dir = None - plot_results(wrms_results, true_pos_results, selected_error, obs_array, data_dir=data_dir, *args, **kwargs) + hf = plot_results(wrms_results, true_pos_results, selected_error, obs_array, data_dir=data_dir, *args, **kwargs) plt.show() + return hf if __name__ == "__main__": - load_and_plot(save_plots=True, bars=True) \ No newline at end of file + hf = load_and_plot(save_plots=False, bars=True) \ No newline at end of file diff --git a/plot_tools.py b/plot_tools.py index fadff11..9b38bb7 100644 --- a/plot_tools.py +++ b/plot_tools.py @@ -3,6 +3,9 @@ import matplotlib.pyplot as plt from matplotlib.patches import Polygon from nice_plot_colors import * +from cycler import cycler + +plt.rc('axes', prop_cycle=(cycler('color', [greyify(c, .5, .8) for c in reversed(lines)]))) def make_poly_array(x,y,sigma): nx = len(x) @@ -14,9 +17,15 @@ def make_poly_array(x,y,sigma): def plot_with_bounds(ax, x, y, s, c=lines[0]): - xy = make_poly_array(x, y, s) + isort = np.argsort(x.flat) + xx, yy = x[isort], y[isort] + try: + ss = s[isort] + except: + ss = s + xy = make_poly_array(xx, yy, ss) h_patch = Polygon(xy, ec=c, fc=lighten(c, 3), alpha=0.5) - h_fx, = ax.plot(x, y, lw=1.5, c=c) + h_fx, = ax.plot(xx, yy, lw=1.5, c=c) ax.add_patch(h_patch) clim = ax.get_ylim() ax.set_ylim(bottom = min(clim[0],xy[:,1].min()), top = max(clim[1], xy[:,1].max())) @@ -123,13 +132,19 @@ def true_plots_rel(xt, ft, mu_t, rel_sigma, y_samples, p_a_y, p_r_y, xa_train, y def estimate_plots(xt, ft, mu_t, fhat, vhat, E_y, rel_sigma, y_samples, p_a_y, p_r_y, xa_train, ya_train, uvr_train, fuvr_train, yr_train, - class_icons = ['ko', 'wo'], marker_options = {'mec':'k', 'mew':0.5}, *args, **kwargs): + class_icons = ['ko', 'wo'], marker_options = {'mec':'k', 'mew':0.5}, n_posterior_samples=0, *args, **kwargs): fig, (ax_l, ax_a, ax_r) = plot_setup_2d(**kwargs) + # Posterior samples + if n_posterior_samples > 0: + y_post = np.random.multivariate_normal(fhat.flatten(), vhat, n_posterior_samples) + h_pp = ax_l.plot(xt, y_post.T, lw=0.8) + # Latent function hf, hpf = plot_with_bounds(ax_l, xt, ft, rel_sigma, c=lines[0]) hf_hat, hpf_hat = plot_with_bounds(ax_l, xt, fhat, np.sqrt(np.atleast_2d(vhat.diagonal()).T), c=lines[1]) + ax_l.legend([hf, hf_hat], [r'True latent function, $f(x)$', r'$\mathcal{GP}$ estimate $\hat{f}(x)$']) # Absolute posterior likelihood diff --git a/predictive_entropy_search.py b/predictive_entropy_search.py new file mode 100644 index 0000000..e69de29 diff --git a/pref_active_learning.py b/pref_active_learning.py index fbea583..8f0eab4 100644 --- a/pref_active_learning.py +++ b/pref_active_learning.py @@ -12,12 +12,12 @@ plt.rc('font',**{'family':'serif','sans-serif':['Computer Modern Roman']}) plt.rc('text', usetex=True) -save_plots = True +save_plots = False -# log_hyp = np.log([0.1,0.5,0.1,10.0]) # length_scale, sigma_f, sigma_probit, v_beta +# log_hyp = np.log([0.1,0.5,0.1,10.0]) # length_scale/s, sigma_f, sigma_n_rel, sigma_beta, v_beta # log_hyp = np.log([0.07, 0.75, 0.25, 1.0, 28.1]) # log_hyp = np.log([0.05, 1.5, 0.09, 2.0, 50.0]) -log_hyp = np.log([0.02, 0.6, 0.2, 0.8, 60.0]) +log_hyp = np.log([0.02, 0.5, 0.1, 0.8, 60.0]) np.random.seed(2) n_rel_train = 1 @@ -32,7 +32,7 @@ n_mcsamples = 1000 n_ysamples = 101 -n_queries = 20 +n_queries = 10 # Define polynomial function to be modelled # true_function = test_data.multi_peak @@ -78,13 +78,19 @@ fig_t.savefig(fig_dir+'true.pdf', bbox_inches='tight') # Construct active learner object -learner = active_learners.UCBAbsRel(x_rel, uvi_rel, x_abs, y_rel, y_abs, delta_f=delta_f, +# learner = active_learners.UCBAbsRelD(x_rel, uvi_rel, x_abs, y_rel, y_abs, delta_f=delta_f, +# rel_likelihood=GPpref.PrefProbit(), abs_likelihood=GPpref.AbsBoundProbit()) +# # obs_arguments = {'req_improvement': 0.60, 'n_test': 50, 'gamma': 2.0, 'n_rel_samples': 5, 'p_thresh': 0.7} +# obs_arguments = {'n_test': 100, 'p_rel': 0.5, 'n_rel_samples': 5, 'gamma': 2.0} + +learner = active_learners.ExpectedImprovementRel(x_rel, uvi_rel, x_abs, y_rel, y_abs, delta_f=delta_f, rel_likelihood=GPpref.PrefProbit(), abs_likelihood=GPpref.AbsBoundProbit()) -# obs_arguments = {'req_improvement': 0.60, 'n_test': 50, 'gamma': 2.0, 'n_rel_samples': 5, 'p_thresh': 0.7} -obs_arguments = {'n_test': 100, 'p_rel': 0.5, 'n_rel_samples': 5, 'gamma': 2.0} +obs_arguments = {'n_test': 100, 'zeta': 0.1} + # Get initial solution learner.set_hyperparameters(log_hyp) f = learner.solve_laplace() +learner.print_hyperparameters() if save_plots: fig_p, (ax_p_l, ax_p_a, ax_p_r) = learner.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, @@ -93,13 +99,14 @@ # fig_p.savefig(fig_dir+'posterior00.pdf', bbox_inches='tight') for obs_num in range(n_queries): + obs_arguments['p_rel'] = max(0.0, (n_queries - obs_num) / float(n_queries)) next_x = learner.select_observation(**obs_arguments) if next_x.shape[0] == 1: next_y, next_f = abs_obs_fun.generate_observations(next_x) learner.add_observations(next_x, next_y) print 'Abs: x:{0}, y:{1}'.format(next_x[0], next_y[0]) else: - next_y, next_uvi, next_fx = rel_obs_fun.cheat_multi_sampler(next_x) + next_y, next_uvi, next_fx = rel_obs_fun.gaussian_multi_pairwise_sampler(next_x) next_fuv = next_fx[next_uvi][:, :, 0] fuv_rel = np.concatenate((fuv_rel, next_fuv), 0) learner.add_observations(next_x, next_y, next_uvi) diff --git a/test_data.py b/test_data.py index cd5a351..a29ff29 100644 --- a/test_data.py +++ b/test_data.py @@ -1,4 +1,5 @@ import numpy as np +import pickle class ObsObject(object): @@ -55,6 +56,9 @@ def randomize(self, print_vals=False): def print_values(self): print "a: {0}, f: {1}, o: {2}, d: {3}".format(self.amplitude, self.frequency, self.offset, self.damping) + def get_values(self): + return self.amplitude, self.frequency, self.offset, self.damping + class DoubleMultiWave(object): def __init__(self, amp_range, f_range, off_range, damp_range, n_components=2): @@ -76,6 +80,23 @@ def print_values(self): self.f_high.print_values() +class WaveSaver(object): + def __init__(self, n_trials, n_components): + self.amplitude = np.zeros((n_trials, n_components), dtype='float') + self.frequency = np.zeros((n_trials, n_components), dtype='float') + self.offset = np.zeros((n_trials, n_components), dtype='float') + self.damping = np.zeros((n_trials, n_components), dtype='float') + + def set_vals(self, n, a, f, o, d): + self.amplitude[n] = a + self.frequency[n] = f + self.offset[n] = o + self.damping[n] = d + + def save(self, filename): + with open(filename, 'wb') as fh: + pickle.dump(self, fh) + def damped_wave(x): y = np.cos(6 * np.pi * (x - 0.5)) * np.exp(-10 * (x - 0.5) ** 2) return y From 37e71df300c231e6397253ea29949ee7e9200015 Mon Sep 17 00:00:00 2001 From: Nicholas Lawrance Date: Thu, 5 Oct 2017 18:06:58 -0700 Subject: [PATCH 15/38] Added YAML stuff for saving trial data, trying to make statruns work with new learners --- GP_preference_demo.py | 25 ++++---- GPpref.py | 19 ++++-- active_learners.py | 137 +++++++++++++++++++++++++++------------- active_statruns.py | 2 + pref_active_learning.py | 24 ++++--- test_data.py | 3 + 6 files changed, 139 insertions(+), 71 deletions(-) diff --git a/GP_preference_demo.py b/GP_preference_demo.py index b06dcf4..7811e81 100644 --- a/GP_preference_demo.py +++ b/GP_preference_demo.py @@ -5,31 +5,30 @@ import scipy.optimize as op import plot_tools as ptt import test_data +import yaml # from scipy.stats import beta plt.rc('font',**{'family':'serif','sans-serif':['Computer Modern Roman']}) plt.rc('text', usetex=True) -np.random.seed(1) +np.random.seed(2) train_hyper = False use_test_data = False verbose = True -#log_hyp = np.log([0.2, 0.5, 0.1, 1.0, 10.0]) # length_scale/s, sigma_f, sigma_n_rel, sigma_beta, v_beta +with open('./data/mid_freq_3.yml', 'rt') as fh: + wave = yaml.safe_load(fh) -# random_wave = test_data.MultiWave(amp_range=[0.6, 1.2], f_range=[10.0, 30.0], off_range=[0.1, 0.9], damp_range=[250.0, 350.0], n_components=3) -# log_hyp = np.log([0.018, 1.0, 0.1, 0.6, 60.0]) +random_wave = test_data.MultiWave(**wave['wave_params']) +log_hyp = np.log(wave['hyperparameters']) -random_wave = test_data.MultiWave(amp_range=[1.2, 3.0], f_range=[5.0, 8.0], off_range=[0.1, 0.9], damp_range=[30.0, 50.0], n_components=2) -log_hyp = np.log([0.1, 2.0, 0.1, 5.0, 30.0]) +n_rel_train = 30 +n_abs_train = 30 -n_rel_train = 10 -n_abs_train = 0 - -rel_sigma = 0.1 +rel_sigma = 0.05 delta_f = 1e-5 -beta_sigma = 1.5 -beta_v = 40.0 +beta_sigma = 0.8 +beta_v = 80.0 n_xplot = 101 n_mcsamples = 1000 @@ -51,7 +50,7 @@ # Main program # True function -x_plot = np.linspace(-1.0,1.0,n_xplot,dtype='float') +x_plot = np.linspace(0.0,1.0,n_xplot,dtype='float') x_test = np.atleast_2d(x_plot).T f_true = abs_obs_fun.f(x_test) mu_true = abs_obs_fun.mean_link(x_test) diff --git a/GPpref.py b/GPpref.py index ed8c4fe..74e132a 100644 --- a/GPpref.py +++ b/GPpref.py @@ -271,16 +271,23 @@ def set_observations(self, x_rel, uvi_rel, x_abs, y_rel, y_abs): self._nx = self.x_train_all.shape[0] self.Ix = np.eye(self._nx) - def add_observations(self, x, y, uvi=None): + def add_observations(self, x, y, uvi=None, keep_f=False): + # keep_f is used to reset the Laplace solution. If it's a small update, it's sometimes better to keep the old + # values and append some 0's for the new observations (keep_f = True), otherwise it is reset (keep_f = False) + # Default is keep_f = False if uvi is None: - x_abs = np.concatenate((self.x_abs, x), 0) - y_abs = np.concatenate((self.y_abs, y), 0) + x_abs = np.concatenate((self.x_abs, np.atleast_2d(x)), 0) + y_abs = np.concatenate((self.y_abs, np.atleast_2d(y)), 0) self.set_observations(self.x_rel, self.uvi_rel, x_abs, self.y_rel, y_abs) else: - x_rel = np.concatenate((self.x_rel, x), 0) - y_rel = np.concatenate((self.y_rel, y), 0) + x_rel = np.concatenate((self.x_rel, np.atleast_2d(x)), 0) + y_rel = np.concatenate((self.y_rel, np.atleast_2d(y)), 0) uvi_rel = np.concatenate((self.uvi_rel, uvi + self.x_rel.shape[0]), 0) self.set_observations(x_rel, uvi_rel, self.x_abs, y_rel, self.y_abs) + if not keep_f: + self.f = None + else: + self.f = np.vstack((self.f, np.zeros((x.shape[0], 1)))) def calc_laplace(self, loghyp): self.kern.lengthscale = np.exp(loghyp[0:self._xdim]) @@ -289,7 +296,7 @@ def calc_laplace(self, loghyp): self.abs_likelihood.set_sigma(np.exp(loghyp[-2])) # I think this sigma relates to sigma_f in the covariance, and is actually possibly redundant self.abs_likelihood.set_v(np.exp(loghyp[-1])) # Should this relate to the rel_likelihood probit noise? - if self.f is None or self.f.shape[0] is not self._nx: + if self.f is None or self.f.shape[0] is not self._nx or np.isnan(self.f).any(): f = np.zeros((self._nx, 1), dtype='float') else: f = self.f diff --git a/active_learners.py b/active_learners.py index 3912c2a..277e190 100644 --- a/active_learners.py +++ b/active_learners.py @@ -2,17 +2,20 @@ import GPpref from scipy.stats import beta import plot_tools as ptt +import time def calc_ucb(fhat, vhat, gamma=2.0, sigma_offset=0.0): return fhat + gamma * (np.sqrt(np.atleast_2d(vhat.diagonal()).T) - sigma_offset) + def softmax_selector(x, tau=1.0): # High tau is more random ex = np.exp((x - x.max())/tau) Px = ex/ex.sum() return np.random.choice(len(x), p=Px) + class ActiveLearner(GPpref.PreferenceGaussianProcess): def init_extras(self): self._default_uvi = np.array([[0, 1]]) @@ -74,6 +77,7 @@ def create_posterior_plot(self, x_test, f_true, mu_true, rel_sigma, fuv_train, a t_r=r'Posterior relative likelihood $P(x_0 \succ x_1 | \mathcal{Y}, \theta)$') return fig_p, (ax_p_l, ax_p_a, ax_p_r) + class UCBLatent(ActiveLearner): # All absolute returns def select_observation(self, domain=None, n_test=100, gamma=2.0): @@ -82,6 +86,7 @@ def select_observation(self, domain=None, n_test=100, gamma=2.0): ucb = calc_ucb(fhat, vhat, gamma) return x_test[[np.argmax(ucb)], :] + class UCBLatentSoftmax(ActiveLearner): # All absolute returns def select_observation(self, domain=None, n_test=100, gamma=2.0, tau=1.0): @@ -90,6 +95,7 @@ def select_observation(self, domain=None, n_test=100, gamma=2.0, tau=1.0): ucb = calc_ucb(fhat, vhat, gamma).flatten() return x_test[[softmax_selector(ucb, tau)], :] + class UCBOut(ActiveLearner): # NOT FULLY IMPLEMENTED - BROKEN def select_observation(self, domain=None, n_test=100, gamma=2.0): @@ -172,6 +178,7 @@ def select_observation(self, domain=None, n_test=100, p_thresh=0.7): # ucb = calc_ucb(fhat, vhat, gamma) return x_test[[np.argmax(p_under_thresh * (1.0 - p_under_thresh))], :] + class UCBAbsRel(ActiveLearner): def select_observation(self, domain=None, n_test=100, p_rel=0.5, n_rel_samples=2, gamma=2.0, tau=1.0): x_test = self.uniform_domain_sampler(n_test, domain) @@ -193,6 +200,7 @@ def select_observation(self, domain=None, n_test=100, p_rel=0.5, n_rel_samples=2 best_n = [softmax_selector(ucb, tau=tau/2.0)] #[np.argmax(ucb)] # return x_test[best_n, :] + class UCBAbsRelD(ActiveLearner): def select_observation(self, domain=None, n_test=100, p_rel=0.5, n_rel_samples=2, gamma=2.0, tau=1.0): x_test = self.uniform_domain_sampler(n_test, domain) @@ -219,18 +227,21 @@ class PeakComparitor(ActiveLearner): def test_observation(self, x, y, x_test, gamma): self.store_observations() self.add_observations(x, y, self._default_uvi) - f = self.solve_laplace() + self.solve_laplace() fhat, vhat = self.predict_latent(x_test) ucb = calc_ucb(fhat, vhat, gamma) self.reset_observations() return ucb.max() def store_observations(self): - self.crx, self.cuv, self.cax, self.cry, self.cay = self.get_observations() + crx, cuv, cax, cry, cay = self.get_observations() + self.crx, self.cuv, self.cax, self.cry, self.cay = crx.copy(), cuv.copy(), cax.copy(), cry.copy(), cay.copy() + self.crf = self.f.copy() def reset_observations(self): try: self.set_observations(self.crx, self.cuv, self.cax, self.cry, self.cay) + self.f = self.crf except AttributeError: print "reset_observations failed: existing observations not found" @@ -304,7 +315,8 @@ def select_observation(self, domain=None, n_test=50, req_improvement=0.6, n_rel_ Vmax = V.max() # best_n = np.argpartition(V, -n_comparators)[-n_comparators:] # best = np.argmax(V) - print 'V_max = {0}'.format(Vmax) + if self.verbose: + print 'V_max = {0}'.format(Vmax) if Vmax < req_improvement: # aa, bb = self.abs_likelihood.get_alpha_beta(fhat) @@ -332,64 +344,103 @@ def calculate_threshold_utility(self, fhat, vhat, n_samples, y_threshold): threshold_utility = 0.0 # For each one, evaluate the probability mass above the threshold in the absolute function for yi in y_post: - threshold_utility += self.mean_P_above_threshold(yi, y_threshold) + threshold_utility += self.point_utility(yi, y_threshold) return threshold_utility/n_samples - def mean_P_above_threshold(self, f, y_threshold): + def point_utility(self, f, y_threshold): + # This is the mean probability mass above the threshold value pmass = 0.0 for fi in f: pmass += 1.0 - self.abs_likelihood.cdf(y_threshold, fi) return pmass/f.shape[0] - def test_observation(self, x, y, x_test, n_samples, y_threshold): - self.store_observations() - self.add_observations(x, y, self._default_uvi) - util, f, v = self.calculate_threshold_utility(x_test, n_samples, y_threshold) + def test_observation(self, x, y, uvi, x_test, n_samples, y_threshold, f = None): + self.add_observations(x, y, uvi, keep_f=True) + self.solve_laplace() + fhat, vhat = self.predict_latent(x_test) + util = self.calculate_threshold_utility(fhat, vhat, n_samples, y_threshold) self.reset_observations() return util - def select_observation(self, domain=None, x_test=None, n_test=100, n_samples=100, y_threshold=0.8): + def select_observation(self, domain=None, x_test=None, n_test=50, n_samples=50, y_threshold=0.8, p_pref_tol=1e-3, n_mc_abs=5): # Generate a set of test points in the domain (if not specified) if x_test is None: - x_test = self.linear_domain_sampler(n_test, domain) + # x_test = self.linear_domain_sampler(n_test, domain) + x_test = self.uniform_domain_sampler(n_test, domain) + x_test.sort(axis=0) + n_test = len(x_test) # Sample a set of functions from the current posterior - self.solve_laplace() + # We save the f value because otherwise it gets out of whack when we add observations + flap = self.solve_laplace() fhat, vhat = self.predict_latent(x_test) - base_utility = self.calculate_threshold_utility(fhat, vhat, n_samples, y_threshold) - max_xi = np.argmax(fhat) - other_xi = np.delete(np.arange(n_test), max_xi) - uvi = np.vstack((max_xi * np.ones(n_test - 1, dtype='int'), other_xi)).T + # base_utility = self.calculate_threshold_utility(fhat, vhat, n_samples, y_threshold) + + # Check a random set of pairwise relatives and all absolutes (massive sampling) + + # Generate a set of random pairs (this randomly pairs all x_test points) + uvi = np.random.choice(n_test, (n_test/2, 2), replace=False) p_pref = self.rel_likelihood.posterior_likelihood(fhat, vhat, uvi, y=-1) - V = np.zeros(n_test - 1) - x = np.zeros((2, 1), dtype='float') - x[0] = x_test[max_xi] + V_max_rel = 0.0 + + self.store_observations() + t_rel = time.time() # Now calculate the expected value for each observation pair - for i,uvi1 in enumerate(other_xi): - x[1] = x_test[uvi1] - V[i] += p_pref[i]*self.test_observation(x, self._minus_y_obs, x_test, max_xi) - if (1-p_pref[i]) > 1e-3: - V[i] += (1-p_pref[i])*self.test_observation(x, self._plus_y_obs, x_test, max_xi) + for i, uv in enumerate(uvi): + x = x_test[uv] + V_rel = 0.0 + if p_pref[i] < p_pref_tol: + V_rel += (1-p_pref[i])*self.test_observation(x, self._plus_y_obs, self._default_uvi, x_test, n_samples, y_threshold, f=flap) + elif p_pref[i] > 1.0-p_pref_tol: + V_rel += p_pref[i]*self.test_observation(x, self._minus_y_obs, self._default_uvi, x_test, n_samples, y_threshold, f=flap) + else: + V_rel += (1-p_pref[i])*self.test_observation(x, self._plus_y_obs, self._default_uvi, x_test, n_samples, y_threshold, f=flap) + V_rel += p_pref[i] * self.test_observation(x, self._minus_y_obs, self._default_uvi, x_test, n_samples, y_threshold, f=flap) + if V_rel >= V_max_rel: + V_max_rel = V_rel + x_best_rel = x - Vmax = V.max() # best_n = np.argpartition(V, -n_comparators)[-n_comparators:] # best = np.argmax(V) - print 'V_max = {0}'.format(Vmax) - - if Vmax < req_improvement: - # aa, bb = self.abs_likelihood.get_alpha_beta(fhat) - # p_under_thresh = beta.cdf(p_thresh, aa, bb) - # return x_test[[np.argmax(p_under_thresh*(1.0-p_under_thresh))], :] - ucb = calc_ucb(fhat, vhat, gamma, self.rel_likelihood.sigma) - return x_test[[np.argmax(ucb)], :] - else: - best_n = [] - while len(best_n) < n_comparators: - cbest = np.argmax(V) - best_n.append(cbest) - V = V * np.sqrt(GPpref.squared_distance(x_test[[other_xi[cbest]], :], x_test[other_xi])[0]) - xi = np.zeros(n_comparators+1, dtype='int') - xi[0] = max_xi - xi[1:] = other_xi[best_n] - return x_test[xi, :] \ No newline at end of file + if self.verbose: + print 'V_max_rel = {0}, x = {2}, t = {1}s'.format(V_max_rel[0], time.time()-t_rel, x_best_rel[:,0]) + + V_max = 0.0 + t_rel = time.time() + for i, x in enumerate(x_test): + F = fhat[i] + np.random.randn(n_mc_abs)*np.sqrt(vhat[i,i]) + Y, mu = self.abs_likelihood.generate_samples(F) + V_abs = 0.0 + for y in Y: + try: + V_abs += self.test_observation(x, y, None, x_test, n_samples, y_threshold, f=flap) + except ValueError: + print "NaN Issue" + V_abs /= n_mc_abs + if V_abs > V_max: + V_max = V_abs + x_best = x + if self.verbose: + print 'V_max_abs = {0}, x = {2}, t = {1}s'.format(V_max, time.time() - t_rel, x_best) + + if V_max_rel > V_max: + x_best = x_best_rel + + return x_best + + +class SampledClassification(SampledThreshold): + def point_utility(self, f, y_threshold): + # Class accuracy + # Note this assumes the sampled functions are actual functions (not the actual gaussian posterior distribution, + # which might be a good alternative, but it isn't this method) + fm = self.abs_likelihood.mean_link(f) + fa = self.abs_likelihood.v * fm + fb = self.abs_likelihood.v - fa + P_below = GPpref.beta.cdf(y_threshold, fa, fb) + + predict_below = fm < y_threshold + predicted_accuracy = P_below[predict_below].sum() + (1.0 - P_below[~predict_below]).sum() + + return predicted_accuracy/len(f) diff --git a/active_statruns.py b/active_statruns.py index 34f0033..b0b8a63 100644 --- a/active_statruns.py +++ b/active_statruns.py @@ -85,6 +85,8 @@ def wrms2(y_true, y_est): # Learner(ABSThresh, {'n_test': 100, 'p_thresh': 0.7}), # 'ABSThresh' # Learner(PeakComparitor, {'gamma': 2.0, 'n_test': 50, 'n_rel_samples': n_rel_samples}), # 'PeakComparitor' # Learner(LikelihoodImprovement, {'req_improvement': 0.60, 'n_test': 50, 'gamma': 2.0, 'n_rel_samples': n_rel_samples, 'p_thresh': 0.7}) # 'LikelihoodImprovement' + Learner(active_learners.SampledThreshold, {'n_test':50, 'n_samples':10, 'y_threshold':0.8, 'p_pref_tol':1e-3, 'n_mc_abs':5}), # 'SampledThreshold' + Learner(active_learners.SampledClassification, {'n_test':50, 'n_samples':10, 'y_threshold':0.8, 'p_pref_tol':1e-3, 'n_mc_abs':5}), # 'SampledClassification' ] names = ['Random (rel and abs)','Random (rel only)', 'Random (abs only)', 'UCB (abs only)', 'EI (Abs)', 'EI (Rel)', 'UCBCombined (rel and abs)'] assert len(names) == len(learners), "Number of names does not match number of learners." diff --git a/pref_active_learning.py b/pref_active_learning.py index 8f0eab4..b3cc99d 100644 --- a/pref_active_learning.py +++ b/pref_active_learning.py @@ -12,7 +12,7 @@ plt.rc('font',**{'family':'serif','sans-serif':['Computer Modern Roman']}) plt.rc('text', usetex=True) -save_plots = False +save_plots = True # log_hyp = np.log([0.1,0.5,0.1,10.0]) # length_scale/s, sigma_f, sigma_n_rel, sigma_beta, v_beta # log_hyp = np.log([0.07, 0.75, 0.25, 1.0, 28.1]) @@ -20,7 +20,7 @@ log_hyp = np.log([0.02, 0.5, 0.1, 0.8, 60.0]) np.random.seed(2) -n_rel_train = 1 +n_rel_train = 2 n_abs_train = 1 rel_sigma = 0.05 delta_f = 1e-5 @@ -32,7 +32,7 @@ n_mcsamples = 1000 n_ysamples = 101 -n_queries = 10 +n_queries = 30 # Define polynomial function to be modelled # true_function = test_data.multi_peak @@ -78,14 +78,18 @@ fig_t.savefig(fig_dir+'true.pdf', bbox_inches='tight') # Construct active learner object -# learner = active_learners.UCBAbsRelD(x_rel, uvi_rel, x_abs, y_rel, y_abs, delta_f=delta_f, -# rel_likelihood=GPpref.PrefProbit(), abs_likelihood=GPpref.AbsBoundProbit()) +GP_kwargs = {'x_rel':x_rel, 'uvi_rel':uvi_rel, 'x_abs':x_abs, 'y_rel':y_rel, 'y_abs':y_abs, 'delta_f':delta_f, + 'rel_likelihood':GPpref.PrefProbit(), 'abs_likelihood':GPpref.AbsBoundProbit()} + +# learner = active_learners.UCBAbsRelD(**active_args) # # obs_arguments = {'req_improvement': 0.60, 'n_test': 50, 'gamma': 2.0, 'n_rel_samples': 5, 'p_thresh': 0.7} # obs_arguments = {'n_test': 100, 'p_rel': 0.5, 'n_rel_samples': 5, 'gamma': 2.0} -learner = active_learners.ExpectedImprovementRel(x_rel, uvi_rel, x_abs, y_rel, y_abs, delta_f=delta_f, - rel_likelihood=GPpref.PrefProbit(), abs_likelihood=GPpref.AbsBoundProbit()) -obs_arguments = {'n_test': 100, 'zeta': 0.1} +# learner = active_learners.ExpectedImprovementRel(**GP_kwargs) +# obs_arguments = {'n_test': 100, 'zeta': 0.1, 'p_rel':1.0} + +learner = active_learners.SampledClassification(**GP_kwargs) +obs_arguments = {'n_test':50, 'n_samples':10, 'y_threshold':0.8, 'p_pref_tol':1e-3, 'n_mc_abs':5} # Get initial solution learner.set_hyperparameters(log_hyp) @@ -99,7 +103,8 @@ # fig_p.savefig(fig_dir+'posterior00.pdf', bbox_inches='tight') for obs_num in range(n_queries): - obs_arguments['p_rel'] = max(0.0, (n_queries - obs_num) / float(n_queries)) + if 'p_rel' in obs_arguments: + obs_arguments['p_rel'] = max(0.0, (n_queries - obs_num) / float(n_queries)) next_x = learner.select_observation(**obs_arguments) if next_x.shape[0] == 1: next_y, next_f = abs_obs_fun.generate_observations(next_x) @@ -129,3 +134,4 @@ pdf_pages.close() plt.show() +print "Finished!" \ No newline at end of file diff --git a/test_data.py b/test_data.py index a29ff29..fc90a17 100644 --- a/test_data.py +++ b/test_data.py @@ -37,6 +37,9 @@ def randomize(self, print_vals=False): def print_values(self): print "a: {0:.2f}, f: {1:.2f}, o: {2:.2f}, d: {3:.2f}".format(self.amplitude, self.frequency, self.offset, self.damping) + def export_state(self): + return dict(amplitude=self.amplitude, frequency=self.frequency, offset=self.offset, damping=self.damping) + class MultiWave(VariableWave): def out(self, x): From 2b27140705c1f69500d0fdf9063aa54494fea6d1 Mon Sep 17 00:00:00 2001 From: Nicholas Lawrance Date: Fri, 27 Oct 2017 14:00:36 -0700 Subject: [PATCH 16/38] Added more learners, currently switching to have learners as a list in the yaml --- GP_preference_demo.py | 44 +++++----- GPpref.py | 29 +++--- active_learners.py | 179 ++++++++++++++++++++++++++++--------- active_statruns.py | 190 +++++++++++++++++++++------------------- plot_statruns.py | 25 ++++-- plot_tools.py | 3 +- pref_active_learning.py | 73 ++++++++------- test_data.py | 77 ++++++++++++++++ 8 files changed, 414 insertions(+), 206 deletions(-) diff --git a/GP_preference_demo.py b/GP_preference_demo.py index 7811e81..afd44c4 100644 --- a/GP_preference_demo.py +++ b/GP_preference_demo.py @@ -9,44 +9,36 @@ # from scipy.stats import beta plt.rc('font',**{'family':'serif','sans-serif':['Computer Modern Roman']}) plt.rc('text', usetex=True) -np.random.seed(2) train_hyper = False -use_test_data = False -verbose = True +use_test_data = False # test_data.data3 # +verbose = 2 -with open('./data/mid_freq_3.yml', 'rt') as fh: +with open('./data/mid_freq_5.yaml', 'rt') as fh: wave = yaml.safe_load(fh) +try: + np.random.seed(wave['statrun_params']['randseed']) +except KeyError: + np.random.seed(0) random_wave = test_data.MultiWave(**wave['wave_params']) log_hyp = np.log(wave['hyperparameters']) -n_rel_train = 30 -n_abs_train = 30 +n_rel_train = 5 +n_abs_train = 5 -rel_sigma = 0.05 delta_f = 1e-5 -beta_sigma = 0.8 -beta_v = 80.0 - n_xplot = 101 n_mcsamples = 1000 -n_ysamples = 101 +n_ysamples = 301 n_posterior_samples = 3 -# Define polynomial function to be modelled -# random_wave = test_data.VariableWave(amp_range=[0.6, 1.2], f_range=[5.0, 10.0], off_range=[0.2, 0.8], -# damp_range=[30.0, 100.0]) -# random_wave.set_values(a=1.2, f=6.0, o=.2, d=20.0) -# random_wave = test_data.DoubleMultiWave(amp_range=[0.0, 0.4, 0.6, 1.2], f_range=[0.1, 2.0, 10.0, 30.0], off_range=[0.0, 1.0, 0.1, 0.9], damp_range=[0.0, 50.0, 250.0, 350.0], n_components=3) -# true_function = test_data.damped_wave - random_wave.print_values() true_function = random_wave.out -rel_obs_fun = GPpref.RelObservationSampler(true_function, GPpref.PrefProbit(sigma=rel_sigma)) -abs_obs_fun = GPpref.AbsObservationSampler(true_function, GPpref.AbsBoundProbit(sigma=beta_sigma, v=beta_v)) +rel_obs_fun = GPpref.RelObservationSampler(true_function, GPpref.PrefProbit(**wave['rel_obs_params'])) +abs_obs_fun = GPpref.AbsObservationSampler(true_function, GPpref.AbsBoundProbit(**wave['abs_obs_params'])) # Main program # True function @@ -63,7 +55,7 @@ # are actually indexes into x, because it is easier computationally. You can # recover the actual u,v values using x[ui],x[vi] if use_test_data: - x_rel, uvi_rel, uv_rel, y_rel, fuv_rel, x_abs, y_abs, mu_abs = test_data.data1() + x_rel, uvi_rel, uv_rel, y_rel, fuv_rel, x_abs, y_abs, mu_abs = use_test_data() else: x_rel, uvi_rel, uv_rel, y_rel, fuv_rel = rel_obs_fun.generate_n_observations(n_rel_train) x_abs, y_abs, mu_abs = abs_obs_fun.generate_n_observations(n_abs_train) @@ -74,6 +66,7 @@ rel_likelihood=GPpref.PrefProbit(), abs_likelihood=GPpref.AbsBoundProbit(), verbose=verbose) +prefGP.set_hyperparameters(log_hyp) # If training hyperparameters, use external optimiser if train_hyper: log_hyp = op.fmin(prefGP.calc_nlml,log_hyp) @@ -93,19 +86,24 @@ # Plot true functions -fig_t, (ax_t_l, ax_t_a, ax_t_r) = ptt.true_plots(x_test, f_true, mu_true, rel_sigma, +fig_t, (ax_t_l, ax_t_a, ax_t_r) = ptt.true_plots(x_test, f_true, mu_true, wave['rel_obs_params']['sigma'], abs_y_samples, p_abs_y_true, p_rel_y_true, x_abs, y_abs, uv_rel, fuv_rel, y_rel, t_l=r'True latent function, $f(x)$') # Posterior estimates fig_p, (ax_p_l, ax_p_a, ax_p_r) = \ - ptt.estimate_plots(x_test, f_true, mu_true, fhat, vhat, E_y, rel_sigma, + ptt.estimate_plots(x_test, f_true, mu_true, fhat, vhat, E_y, wave['rel_obs_params']['sigma'], abs_y_samples, p_abs_y_post, p_rel_y_post, x_abs, y_abs, uv_rel, fuv_rel, y_rel, n_posterior_samples=n_posterior_samples, t_a=r'Posterior absolute likelihood, $p(u | \mathcal{Y}, \theta)$', t_r=r'Posterior relative likelihood $P(x_0 \succ x_1 | \mathcal{Y}, \theta)$') +wrms = test_data.wrms(mu_true, E_y) +wrms2 = test_data.wrms_misclass(mu_true, E_y) +p_err = test_data.rel_error(mu_true, p_rel_y_true, E_y, p_rel_y_post, weight=False) +print "WRMS: {0:0.3f}, WRMS_MC: {1:0.3f}, p_err: {2:0.3f}".format(wrms, wrms2, p_err) + plt.show() diff --git a/GPpref.py b/GPpref.py index 74e132a..0486b06 100644 --- a/GPpref.py +++ b/GPpref.py @@ -6,6 +6,9 @@ from scipy.stats import norm, beta from scipy.linalg import block_diag +class LaplaceException(Exception): + pass + # Define a standard normal pdf _sqrt_2pi = np.sqrt(2*np.pi) def std_norm_pdf(x): @@ -86,7 +89,7 @@ def print_hyperparameters(self): def z_k(self, y, f, scale=None): if scale is None: scale = self._isqrt2sig - zc = scale * (f[:, 1, None] - f[:, 0, None]) # Weird none is to preserve shape + zc = scale * (f[:, 1, None] - f[:, 0, None]) # Weird None is to preserve shape return y * zc def I_k(self, x, uv): # Jensen and Nielsen @@ -275,27 +278,31 @@ def add_observations(self, x, y, uvi=None, keep_f=False): # keep_f is used to reset the Laplace solution. If it's a small update, it's sometimes better to keep the old # values and append some 0's for the new observations (keep_f = True), otherwise it is reset (keep_f = False) # Default is keep_f = False - if uvi is None: + if uvi is None: # Absolute observation/s + if keep_f: + self.f = np.vstack((self.f, np.zeros((x.shape[0], 1)))) x_abs = np.concatenate((self.x_abs, np.atleast_2d(x)), 0) y_abs = np.concatenate((self.y_abs, np.atleast_2d(y)), 0) self.set_observations(self.x_rel, self.uvi_rel, x_abs, self.y_rel, y_abs) - else: + else: # Relative observation/s + if keep_f: # The rel observations are stored at the front of f[0:_n_rel] + self.f = np.vstack((self.f[0:self._n_rel], np.zeros((x.shape[0], 1)), self.f[self._n_rel:])) x_rel = np.concatenate((self.x_rel, np.atleast_2d(x)), 0) y_rel = np.concatenate((self.y_rel, np.atleast_2d(y)), 0) uvi_rel = np.concatenate((self.uvi_rel, uvi + self.x_rel.shape[0]), 0) self.set_observations(x_rel, uvi_rel, self.x_abs, y_rel, self.y_abs) - if not keep_f: - self.f = None - else: - self.f = np.vstack((self.f, np.zeros((x.shape[0], 1)))) - def calc_laplace(self, loghyp): + def set_hyperparameters(self, loghyp): self.kern.lengthscale = np.exp(loghyp[0:self._xdim]) - self.kern.variance = (np.exp(loghyp[self._xdim]))**2 + self.kern.variance = 1.0 # (np.exp(loghyp[self._xdim]))**2 self.rel_likelihood.set_sigma(np.exp(loghyp[-3])) # Do we need different sigmas for each likelihood? Yes! self.abs_likelihood.set_sigma(np.exp(loghyp[-2])) # I think this sigma relates to sigma_f in the covariance, and is actually possibly redundant self.abs_likelihood.set_v(np.exp(loghyp[-1])) # Should this relate to the rel_likelihood probit noise? + def calc_laplace(self, loghyp=None): + if loghyp is not None: + self.set_hyperparameters(loghyp) + if self.f is None or self.f.shape[0] is not self._nx or np.isnan(self.f).any(): f = np.zeros((self._nx, 1), dtype='float') else: @@ -357,8 +364,8 @@ def calc_laplace(self, loghyp): f = f_new nloops += 1 if nloops > 10000: - raise RuntimeError("Maximum loops exceeded in calc_laplace!!") - if self.verbose: + raise LaplaceException("Maximum loops exceeded in calc_laplace!!") + if self.verbose > 1: lml = py - 0.5*np.matmul(f.T, np.matmul(self.iK, f)) - 0.5*np.log(np.linalg.det(np.matmul(W, self.Kxx) + self.Ix)) print "Laplace iteration {0:02d}, log p(y|f) = {1:0.2f}".format(nloops, lml[0,0]) diff --git a/active_learners.py b/active_learners.py index 277e190..f91c6e3 100644 --- a/active_learners.py +++ b/active_learners.py @@ -6,7 +6,8 @@ def calc_ucb(fhat, vhat, gamma=2.0, sigma_offset=0.0): - return fhat + gamma * (np.sqrt(np.atleast_2d(vhat.diagonal()).T) - sigma_offset) + # return fhat + gamma * (np.sqrt(np.atleast_2d(vhat.diagonal()).T) - sigma_offset) + return fhat.flatten() + gamma * (np.sqrt(np.diagonal(vhat)) - sigma_offset) def softmax_selector(x, tau=1.0): @@ -22,12 +23,7 @@ def init_extras(self): self._plus_y_obs = np.ones((1, 1), dtype='int') self._minus_y_obs = -1*self._plus_y_obs - def set_hyperparameters(self, log_hyp): - self.log_hyp = log_hyp - def solve_laplace(self, log_hyp=None): - if log_hyp is None: - log_hyp = self.log_hyp self.f = self.calc_laplace(log_hyp) return self.f @@ -39,11 +35,13 @@ def select_observation(self, p_rel=0.5, domain=None, n_rel_samples=2): n_rel_samples = 1 return self.uniform_domain_sampler(n_rel_samples, domain) - def uniform_domain_sampler(self, n_samples, domain=None): + def uniform_domain_sampler(self, n_samples, domain=None, sortx=False): # Domain should be 2 x n_xdim, i.e [[x0_lo, x1_lo, ... , xn_lo], [x0_hi, x1_hi, ... , xn_hi ]] x_test = np.random.uniform(size=(n_samples, self._xdim)) if domain is not None: x_test = x_test*np.diff(domain, axis=0) + domain[0, :] + if sortx: + x_test = np.sort(x_test, axis=0) return x_test def linear_domain_sampler(self, n_samples, domain=None): @@ -78,6 +76,14 @@ def create_posterior_plot(self, x_test, f_true, mu_true, rel_sigma, fuv_train, a return fig_p, (ax_p_l, ax_p_a, ax_p_r) +class MaxVar(ActiveLearner): + # Max variance + def select_observation(self, domain=None, n_test=100): + x_test = self.uniform_domain_sampler(n_test, domain) + fhat, vhat = self.predict_latent(x_test) + return x_test[[np.argmax(np.diagonal(vhat))], :] + + class UCBLatent(ActiveLearner): # All absolute returns def select_observation(self, domain=None, n_test=100, gamma=2.0): @@ -92,9 +98,16 @@ class UCBLatentSoftmax(ActiveLearner): def select_observation(self, domain=None, n_test=100, gamma=2.0, tau=1.0): x_test = self.uniform_domain_sampler(n_test, domain) fhat, vhat = self.predict_latent(x_test) - ucb = calc_ucb(fhat, vhat, gamma).flatten() + ucb = calc_ucb(fhat, vhat, gamma) return x_test[[softmax_selector(ucb, tau)], :] +class UCBCovarianceSoftmax(ActiveLearner): + # All absolute returns + def select_observation(self, domain=None, n_test=100, gamma=2.0, tau=1.0): + x_test = self.uniform_domain_sampler(n_test, domain) + fhat, vhat = self.predict_latent(x_test) + ucb = fhat.flatten() + gamma * (vhat.sum(axis=1) ** 0.25) + return x_test[[softmax_selector(ucb, tau)], :] class UCBOut(ActiveLearner): # NOT FULLY IMPLEMENTED - BROKEN @@ -166,7 +179,7 @@ def select_observation(self, domain=None, n_test=100, zeta=0.0, *args, **kwargs) # # x_test = self.uniform_domain_sampler(n_test, domain) # fhat, vhat = self.predict_latent(x_test) -# ucb = calc_ucb(fhat, vhat, gamma).flatten() +# ucb = calc_ucb(fhat, vhat, gamma) # return x_test[[softmax_selector(ucb, tau)], :] class ABSThresh(ActiveLearner): @@ -183,7 +196,7 @@ class UCBAbsRel(ActiveLearner): def select_observation(self, domain=None, n_test=100, p_rel=0.5, n_rel_samples=2, gamma=2.0, tau=1.0): x_test = self.uniform_domain_sampler(n_test, domain) fhat, vhat = self.predict_latent(x_test) - ucb = calc_ucb(fhat, vhat, gamma).flatten() + ucb = calc_ucb(fhat, vhat, gamma) if np.random.uniform() < p_rel: # i.e choose a relative sample best_n = [softmax_selector(ucb, tau=tau)] #[np.argmax(ucb)] # @@ -193,7 +206,7 @@ def select_observation(self, domain=None, n_test=100, p_rel=0.5, n_rel_samples=2 while len(best_n) < n_rel_samples: # ucb = ucb*sq_dist[best_n[-1], :] # Discount ucb by distance ucb[best_n[-1]] = 0.0 - ucb *= 2*p_rel_y[best_n[-1],:]*(1.0 - p_rel_y[best_n[-1],:]) # Divide by likelihood that each point is better than previous best + ucb *= 4*p_rel_y[best_n[-1],:]*(1.0 - p_rel_y[best_n[-1],:]) # Divide by likelihood that each point is better than previous best best_n.append(softmax_selector(ucb, tau=tau)) # best_n.append(np.argmax(ucb)) else: @@ -205,7 +218,7 @@ class UCBAbsRelD(ActiveLearner): def select_observation(self, domain=None, n_test=100, p_rel=0.5, n_rel_samples=2, gamma=2.0, tau=1.0): x_test = self.uniform_domain_sampler(n_test, domain) fhat, vhat = self.predict_latent(x_test) - ucb = calc_ucb(fhat, vhat, gamma).flatten() + ucb = calc_ucb(fhat, vhat, gamma) if np.random.uniform() < p_rel: # i.e choose a relative sample best_n = [softmax_selector(ucb, tau=tau)] #[np.argmax(ucb)] # @@ -214,13 +227,81 @@ def select_observation(self, domain=None, n_test=100, p_rel=0.5, n_rel_samples=2 while len(best_n) < n_rel_samples: # ucb *= sq_dist[best_n[-1], :] # Discount ucb by distance ucb[best_n[-1]] = 0.0 - ucb *= 2*p_rel_y[best_n[-1],:]*(1.0 - p_rel_y[best_n[-1],:]) # Divide by likelihood that each point is better than previous best + ucb *= 4*p_rel_y[best_n[-1],:]*(1.0 - p_rel_y[best_n[-1],:]) # Divide by likelihood that each point is better than previous best best_n.append(softmax_selector(ucb, tau=tau)) #best_n.append(np.argmax(ucb)) else: best_n = [softmax_selector(ucb, tau=tau/2.0)] #[np.argmax(ucb)] # return x_test[best_n, :] +class DetRelBoo(ActiveLearner): + def select_observation(self, domain=None, n_test=100, n_rel_samples=2, gamma=2.0, tau=1.0): + x_test = self.uniform_domain_sampler(n_test, domain, sortx=True) + fhat, vhat = self.predict_latent(x_test) + # ucb = calc_ucb(fhat, vhat, gamma) + + # Select the first location using the mean, uncertainty and likelihood of improvement over everywhere (?) else + p_rel = self.rel_posterior_likelihood_array(fhat=fhat, varhat=vhat) + pp_rel = p_rel.mean(axis=0) + ucb = 4*pp_rel*(1-pp_rel)*np.diagonal(vhat)**0.5 # THIS SEEMS BACKWARD!!! WHY DOES IT WORK? + + available_indexes = set(range(len(x_test))) + dK = np.zeros(len(available_indexes)) + + best_n = [softmax_selector(ucb, tau=tau)] #[np.argmax(ucb)] # + uvi = self._default_uvi.copy() + uvi[0][0] = best_n[0] + while len(best_n) < n_rel_samples: + dK[best_n[-1]] = -1e5 # Bit of a scam because it is still possible to sample this value + available_indexes.remove(best_n[-1]) + best_n.append(-1) + for cn in available_indexes: + best_n[-1] = cn; uvi[0][1] = cn + K = vhat[np.ix_(best_n, best_n)] + # p_y = self.rel_likelihood.posterior_likelihood(fhat, vhat, uvi, y=self._plus_y_obs) + dK[cn] = (gamma*np.sqrt(np.linalg.det(K)) + fhat[cn]) # *p_y + best_n[-1] = softmax_selector(dK, tau=tau) # np.argmax(dK) # + return x_test[best_n, :] + +class DetSelect(ActiveLearner): + def select_observation(self, domain=None, n_test=100, n_rel_samples=2, gamma=2.0, rel_tau=1.0, abs_tau=1.0): + x_test = self.uniform_domain_sampler(n_test, domain, sortx=True) + fhat, vhat = self.predict_latent(x_test) + # ucb = calc_ucb(fhat, vhat, gamma).flatten() + + # Select the first location using the mean, uncertainty and likelihood of improvement over everywhere (?) else + p_rel = self.rel_posterior_likelihood_array(fhat=fhat, varhat=vhat) + pp_rel = p_rel.mean(axis=0) + x_std = np.diagonal(vhat)**0.5 + rel_value = 4*pp_rel*(1-pp_rel)*(x_std.max() - x_std) # fhat.flatten() + gamma*(vhat.sum(axis=1)**0.25) # + + available_indexes = set(range(len(x_test))) + dK = np.zeros(len(available_indexes)) + best_n = [softmax_selector(rel_value, tau=rel_tau)] #[np.argmax(rel_value)] # + uvi = self._default_uvi.copy() + uvi[0][0] = best_n[0] + while len(best_n) < n_rel_samples: + dK[best_n[-1]] = -1e5 # Bit of a scam because it is still possible to sample this value + available_indexes.remove(best_n[-1]) + best_n.append(-1) + for cn in available_indexes: + best_n[-1] = cn; uvi[0][1] = cn + K = vhat[np.ix_(best_n, best_n)] + p_y = self.rel_likelihood.posterior_likelihood(fhat, vhat, uvi, y=self._plus_y_obs) + dK[cn] = p_y*(gamma*np.sqrt(np.linalg.det(K)) + fhat[cn]) + best_n[-1] = softmax_selector(dK, tau=rel_tau) # np.argmax(dK) # + + K = vhat[np.ix_(best_n, best_n)] + mdK = gamma*np.sqrt(np.linalg.det(K)) + fhat[cn] + # mdK = dK.max() + ucb = calc_ucb(fhat, vhat, gamma) + p_rel = mdK/(mdK + ucb.max()) + + if abs_tau > 0 and np.random.uniform() > p_rel: # i.e choose an abs: + # best_n = [softmax_selector(fhat.flatten() + gamma*(vhat.sum(axis=1)**0.25), tau)] #[best_n[0]] # + best_n = [softmax_selector(ucb, tau=abs_tau)] + return x_test[best_n, :] + class PeakComparitor(ActiveLearner): @@ -337,22 +418,21 @@ def select_observation(self, domain=None, n_test=50, req_improvement=0.6, n_rel_ class SampledThreshold(PeakComparitor): - # TODO: FINISH IMPLEMENTATION def calculate_threshold_utility(self, fhat, vhat, n_samples, y_threshold): - y_post = self.sample_latent_posterior(fhat, vhat, n_samples = n_samples) + f_sampled = self.sample_latent_posterior(fhat, vhat, n_samples = n_samples) threshold_utility = 0.0 # For each one, evaluate the probability mass above the threshold in the absolute function - for yi in y_post: - threshold_utility += self.point_utility(yi, y_threshold) + for fs in f_sampled: + threshold_utility += self.point_utility(fhat, fs, y_threshold) return threshold_utility/n_samples - def point_utility(self, f, y_threshold): + def point_utility(self, fhat, fs, y_threshold): # This is the mean probability mass above the threshold value pmass = 0.0 - for fi in f: + for fi in fs: pmass += 1.0 - self.abs_likelihood.cdf(y_threshold, fi) - return pmass/f.shape[0] + return pmass/fs.shape[0] def test_observation(self, x, y, uvi, x_test, n_samples, y_threshold, f = None): self.add_observations(x, y, uvi, keep_f=True) @@ -363,11 +443,14 @@ def test_observation(self, x, y, uvi, x_test, n_samples, y_threshold, f = None): return util def select_observation(self, domain=None, x_test=None, n_test=50, n_samples=50, y_threshold=0.8, p_pref_tol=1e-3, n_mc_abs=5): + # n_test is the number of test point locations on the input function + # n_samples is the number of functions sampled from the posterior for estimating the utility + # n_mc_abs is the number of proposed observations sampled from the current posterior for absolute estimates # Generate a set of test points in the domain (if not specified) if x_test is None: # x_test = self.linear_domain_sampler(n_test, domain) x_test = self.uniform_domain_sampler(n_test, domain) - x_test.sort(axis=0) + # x_test.sort(axis=0) n_test = len(x_test) # Sample a set of functions from the current posterior @@ -385,38 +468,48 @@ def select_observation(self, domain=None, x_test=None, n_test=50, n_samples=50, self.store_observations() + # Relative observations t_rel = time.time() # Now calculate the expected value for each observation pair for i, uv in enumerate(uvi): x = x_test[uv] V_rel = 0.0 - if p_pref[i] < p_pref_tol: - V_rel += (1-p_pref[i])*self.test_observation(x, self._plus_y_obs, self._default_uvi, x_test, n_samples, y_threshold, f=flap) - elif p_pref[i] > 1.0-p_pref_tol: - V_rel += p_pref[i]*self.test_observation(x, self._minus_y_obs, self._default_uvi, x_test, n_samples, y_threshold, f=flap) - else: - V_rel += (1-p_pref[i])*self.test_observation(x, self._plus_y_obs, self._default_uvi, x_test, n_samples, y_threshold, f=flap) - V_rel += p_pref[i] * self.test_observation(x, self._minus_y_obs, self._default_uvi, x_test, n_samples, y_threshold, f=flap) - if V_rel >= V_max_rel: - V_max_rel = V_rel - x_best_rel = x + try: + if p_pref[i] < p_pref_tol: + V_rel += (1-p_pref[i])*self.test_observation(x, self._plus_y_obs, self._default_uvi, x_test, n_samples, y_threshold, f=flap) + elif p_pref[i] > 1.0-p_pref_tol: + V_rel += p_pref[i]*self.test_observation(x, self._minus_y_obs, self._default_uvi, x_test, n_samples, y_threshold, f=flap) + else: + V_rel += (1-p_pref[i])*self.test_observation(x, self._plus_y_obs, self._default_uvi, x_test, n_samples, y_threshold, f=flap) + V_rel += p_pref[i] * self.test_observation(x, self._minus_y_obs, self._default_uvi, x_test, n_samples, y_threshold, f=flap) + if V_rel >= V_max_rel: + V_max_rel = V_rel + x_best_rel = x + except GPpref.LaplaceException as exc: + print "Failed in relative test observation, x = [{0}, {1}]".format(x[0], x[1]) + raise exc # best_n = np.argpartition(V, -n_comparators)[-n_comparators:] # best = np.argmax(V) if self.verbose: print 'V_max_rel = {0}, x = {2}, t = {1}s'.format(V_max_rel[0], time.time()-t_rel, x_best_rel[:,0]) + # Absolute queries V_max = 0.0 t_rel = time.time() for i, x in enumerate(x_test): F = fhat[i] + np.random.randn(n_mc_abs)*np.sqrt(vhat[i,i]) Y, mu = self.abs_likelihood.generate_samples(F) V_abs = 0.0 + Y = np.clip(Y, 1e-2, 1-1e-2) # I had stability problems in Laplace with values approaching 0 or 1 for y in Y: try: V_abs += self.test_observation(x, y, None, x_test, n_samples, y_threshold, f=flap) except ValueError: print "NaN Issue" + except GPpref.LaplaceException as exc: + print "Failed in absolute test observation, x = {0}, y = {1}".format(x, y) + raise exc V_abs /= n_mc_abs if V_abs > V_max: V_max = V_abs @@ -431,16 +524,18 @@ def select_observation(self, domain=None, x_test=None, n_test=50, n_samples=50, class SampledClassification(SampledThreshold): - def point_utility(self, f, y_threshold): + def calculate_threshold_utility(self, fhat, vhat, n_samples, y_threshold): # Class accuracy - # Note this assumes the sampled functions are actual functions (not the actual gaussian posterior distribution, - # which might be a good alternative, but it isn't this method) - fm = self.abs_likelihood.mean_link(f) - fa = self.abs_likelihood.v * fm - fb = self.abs_likelihood.v - fa - P_below = GPpref.beta.cdf(y_threshold, fa, fb) + # Note this assumes the sampled functions are used to make the classification decision, and we calculate the + # probability mass from the current mean estimate above the threshold + P_below = self.abs_likelihood.cdf(y_threshold, fhat) - predict_below = fm < y_threshold - predicted_accuracy = P_below[predict_below].sum() + (1.0 - P_below[~predict_below]).sum() - - return predicted_accuracy/len(f) + f_sampled = self.sample_latent_posterior(fhat, vhat, n_samples=n_samples) + threshold_utility = 0.0 + # For each one, evaluate the probability mass above the threshold in the absolute function + for fs in f_sampled: + fm = self.abs_likelihood.mean_link(fs) + predict_below = fm < y_threshold + predicted_accuracy = P_below[predict_below].sum() + (1.0 - P_below[~predict_below]).sum() + threshold_utility += predicted_accuracy/len(fhat) + return threshold_utility/n_samples \ No newline at end of file diff --git a/active_statruns.py b/active_statruns.py index b0b8a63..6985f9f 100644 --- a/active_statruns.py +++ b/active_statruns.py @@ -4,91 +4,90 @@ import GPpref import plot_tools as ptt import active_learners -# from active_learners import ActiveLearner, UCBLatent, UCBLatentSoftmax, LikelihoodImprovement, ABSThresh, UCBAbsRel, UCBAbsRelD import test_data import pickle import plot_statruns +import yaml + +np.set_printoptions(precision=3) class Learner(object): - def __init__(self, model_type, obs_arguments): - self.model_type = model_type + def __init__(self, model_type, obs_arguments, name): + self.model_type = getattr(active_learners, self.model_type) self.obs_arguments = obs_arguments + self.name = name def build_model(self, training_data): self.model = self.model_type(**training_data) - -def wrms(y_true, y_est, weight=True): - if weight: - w = y_true - else: - w = 1.0 - return np.sqrt(np.mean(((y_true - y_est)*w)**2)) - - -def wrms2(y_true, y_est): - w = np.power(np.maximum(y_true, y_est), 2) - return np.sqrt(np.mean(((y_true - y_est)*w)**2)) - - now_time = time.strftime("%Y_%m_%d-%H_%M") -# log_hyp = np.log([0.1,0.5,0.1,10.0]) # length_scale, sigma_f, sigma_probit, v_beta -# log_hyp = np.log([0.07, 0.75, 0.25, 1.0, 28.1]) -# log_hyp = np.log([0.05, 1.5, 0.09, 2.0, 50.0]) -log_hyp = np.log([0.018, 1.0, 0.2, 0.5, 60.0]) -np.random.seed(1) - -n_rel_train = 1 -n_abs_train = 0 -rel_sigma = 0.05 -delta_f = 1e-5 +with open('./data/statruns2_oct2017.yaml', 'rt') as fh: + statrun_params = yaml.safe_load(fh) -beta_sigma = 0.5 -beta_v = 80.0 +log_hyp = np.log(statrun_params['hyperparameters']) -n_xtest = 101 -n_best_points = 15 +# Statrun parameters +np.random.seed(statrun_params['statrun_params']['randseed']) +n_rel_train = statrun_params['statrun_params']['n_rel_train'] +n_abs_train = statrun_params['statrun_params']['n_abs_train'] +n_xtest = statrun_params['statrun_params']['n_xtest'] +n_best_points = statrun_params['statrun_params']['n_best_points'] +n_trials = statrun_params['statrun_params']['n_trials'] +n_queries = statrun_params['statrun_params']['n_queries'] +if 'calc_relative_error' in statrun_params['statrun_params']: + calc_relative_error = statrun_params['statrun_params']['calc_relative_error'] +else: + calc_relative_error = False -# n_ysamples = 101 -n_trials = 100 -randomize_waves = True -n_rel_samples = 5 +# Learner parameters +n_rel_samples = statrun_params['learner_params']['n_rel_samples'] +delta_f = statrun_params['learner_params']['delta_f'] -n_queries = 80 # Define polynomial function to be modelled -# random_wave = test_data.VariableWave([0.6, 1.0], [5.0, 10.0], [0.0, 1.0], [10.0, 20.0]) -random_wave = test_data.MultiWave(amp_range=[0.6, 1.2], f_range=[10.0, 30.0], off_range=[0.1, 0.9], - damp_range=[250.0, 350.0], n_components=3) +random_wave = test_data.MultiWave(**statrun_params['wave_params']) now_time = time.strftime("%Y_%m_%d-%H_%M") data_dir = 'data/' + now_time + '/' ptt.ensure_dir(data_dir) print "Data will be saved to: {0}".format(data_dir) waver = test_data.WaveSaver(n_trials, random_wave.n_components) +with open(data_dir + 'params.yaml', 'wt') as fh: + yaml.safe_dump(statrun_params, fh) # True function x_plot = np.linspace(0.0, 1.0, n_xtest,dtype='float') x_test = np.atleast_2d(x_plot).T +# Learners +learners = [] +for l in statrun_params['learners']: + learners.append(Learner(**l)) + # Construct active learner object -learners = [Learner(active_learners.ActiveLearner, {'p_rel': 0.5, 'n_rel_samples': n_rel_samples}), # 'Random (rel and abs)', - Learner(active_learners.ActiveLearner, {'p_rel': 1.0, 'n_rel_samples': n_rel_samples}), # 'Random (rel)', - Learner(active_learners.ActiveLearner, {'p_rel': 0.0, 'n_rel_samples': n_rel_samples}), # 'Random (abs)', - Learner(active_learners.UCBLatent, {'gamma': 4.0, 'n_test': 100}), # 'UCBLatent' - # Learner(UCBLatentSoftmax, {'gamma': 2.0, 'n_test': 100, 'tau':1.0}), # 'UCBSoft (abs)', - Learner(active_learners.ExpectedImprovementAbs, { 'n_test': 100 }), # 'EI (Abs)', - Learner(active_learners.ExpectedImprovementRel, { 'n_test': 100 }), # 'EI (Rel)', - Learner(active_learners.UCBAbsRel, { 'n_test': 100, 'n_rel_samples': n_rel_samples, 'gamma': 2.0, 'tau': 1.0}), # 'UCBCombined', - # Learner(UCBAbsRelD, { 'n_test': 100, 'n_rel_samples': n_rel_samples, 'gamma': 2.0, 'tau': 1.0}), # , 'UCBD (rel and abs)' - # Learner(ABSThresh, {'n_test': 100, 'p_thresh': 0.7}), # 'ABSThresh' - # Learner(PeakComparitor, {'gamma': 2.0, 'n_test': 50, 'n_rel_samples': n_rel_samples}), # 'PeakComparitor' - # Learner(LikelihoodImprovement, {'req_improvement': 0.60, 'n_test': 50, 'gamma': 2.0, 'n_rel_samples': n_rel_samples, 'p_thresh': 0.7}) # 'LikelihoodImprovement' - Learner(active_learners.SampledThreshold, {'n_test':50, 'n_samples':10, 'y_threshold':0.8, 'p_pref_tol':1e-3, 'n_mc_abs':5}), # 'SampledThreshold' - Learner(active_learners.SampledClassification, {'n_test':50, 'n_samples':10, 'y_threshold':0.8, 'p_pref_tol':1e-3, 'n_mc_abs':5}), # 'SampledClassification' +learners = [Learner(active_learners.ActiveLearner, {'p_rel': 1.0, 'n_rel_samples': n_rel_samples}, 'Random (rel)'), + Learner(active_learners.ActiveLearner, {'p_rel': 0.0, 'n_rel_samples': n_rel_samples}, 'Random (abs)'), + Learner(active_learners.ActiveLearner, {'p_rel': 0.5, 'n_rel_samples': n_rel_samples}, 'Random ($p_{rel}=0.5$)'), + Learner(active_learners.MaxVar, {'n_test': 100}, 'MaxVar (abs)'), + Learner(active_learners.UCBLatent, {'gamma': 3.0, 'n_test': 100}, 'UCB (abs)'), + Learner(active_learners.UCBLatentSoftmax, {'gamma': 2.0, 'n_test': 100, 'tau':1.0}, 'UCBSoft (abs)'), + Learner(active_learners.UCBCovarianceSoftmax, {'gamma': 2.0, 'n_test': 100}, 'UCBCovSoft (abs)'), + Learner(active_learners.DetRelBoo, {'n_test': 100, 'n_rel_samples': n_rel_samples, 'gamma': 2.0, 'tau': 0.5}, 'DetRelBoo (rel)'), + Learner(active_learners.DetSelect, {'n_test': 100, 'n_rel_samples': n_rel_samples, 'gamma': 2.0, 'abs_tau': -1, 'rel_tau': 1.0e-5}, 'DetSelectRel (rel)'), + Learner(active_learners.DetSelect, {'n_test': 100, 'n_rel_samples': n_rel_samples, 'gamma': 2.0, 'abs_tau': 1.0e-5, 'rel_tau': 1.0e-5}, 'DetSelectGreedy (rel, abs)'), + Learner(active_learners.DetSelect, {'n_test': 100, 'n_rel_samples': n_rel_samples, 'gamma': 2.0, 'abs_tau': 0.5, 'rel_tau': 0.5}, 'DetSelectSoft (rel, abs)'), + Learner(active_learners.ExpectedImprovementAbs, { 'n_test': 100 }, 'EI (Abs)'), + Learner(active_learners.ExpectedImprovementRel, { 'n_test': 100 }, 'EI (Rel)'), + Learner(active_learners.UCBAbsRel, { 'n_test': 100, 'n_rel_samples': n_rel_samples, 'gamma': 2.0, 'tau': 1.0}, 'UCBCombined (rel, abs)'), + # Learner(active_learners.UCBAbsRelD, { 'n_test': 100, 'n_rel_samples': n_rel_samples, 'gamma': 2.0, 'tau': 1.0}, 'UCBD (rel and abs)'), + # Learner(active_learners.ABSThresh, {'n_test': 100, 'p_thresh': 0.7}, 'ABSThresh'), + # Learner(active_learners.PeakComparitor, {'gamma': 2.0, 'n_test': 50, 'n_rel_samples': n_rel_samples}, 'PeakComparitor'), + # Learner(active_learners.LikelihoodImprovement, {'req_improvement': 0.60, 'n_test': 50, 'gamma': 2.0, 'n_rel_samples': n_rel_samples, 'p_thresh': 0.7}, 'LikelihoodImprovement'), + # Learner(active_learners.SampledThreshold, {'n_test':50, 'n_samples':10, 'y_threshold':0.8, 'p_pref_tol':1e-3, 'n_mc_abs':5}, 'SampledThreshold'), + # Learner(active_learners.SampledClassification, {'n_test':50, 'n_samples':10, 'y_threshold':0.8, 'p_pref_tol':1e-3, 'n_mc_abs':5}, 'SampledClassification'), ] -names = ['Random (rel and abs)','Random (rel only)', 'Random (abs only)', 'UCB (abs only)', 'EI (Abs)', 'EI (Rel)', 'UCBCombined (rel and abs)'] +names = [l.name for l in learners] assert len(names) == len(learners), "Number of names does not match number of learners." n_learners = len(learners) @@ -97,6 +96,10 @@ def wrms2(y_true, y_est): wrms_results = np.zeros((n_learners, n_queries+1, n_trials)) true_pos_results = np.zeros((n_learners, n_queries+1, n_trials), dtype='int') selected_error = np.zeros((n_learners, n_queries+1, n_trials)) +if calc_relative_error: + relative_error = np.zeros((n_learners, n_queries+1, n_trials)) +else: + relative_error = None trial_number = 0 # for trial_number in range(n_trials): @@ -104,19 +107,20 @@ def wrms2(y_true, y_est): try: print 'Trial {0}'.format(trial_number) - if randomize_waves: - random_wave.randomize() + random_wave.randomize() random_wave.print_values() waver.set_vals(trial_number, *random_wave.get_values()) - rel_obs_fun = GPpref.RelObservationSampler(random_wave.out, GPpref.PrefProbit(sigma=rel_sigma)) - abs_obs_fun = GPpref.AbsObservationSampler(random_wave.out, GPpref.AbsBoundProbit(sigma=beta_sigma, v=beta_v)) + rel_obs_fun = GPpref.RelObservationSampler(random_wave.out, GPpref.PrefProbit(**statrun_params['rel_obs_params'])) + abs_obs_fun = GPpref.AbsObservationSampler(random_wave.out, GPpref.AbsBoundProbit(**statrun_params['abs_obs_params'])) + f_true = abs_obs_fun.f(x_test) y_abs_true = abs_obs_fun.mean_link(x_test) best_points = np.argpartition(y_abs_true.flatten(), -n_best_points)[-n_best_points:] best_points_set = set(best_points) # abs_y_samples = np.atleast_2d(np.linspace(0.01, 0.99, n_ysamples)).T # p_abs_y_true = abs_obs_fun.observation_likelihood_array(x_test, abs_y_samples) - # p_rel_y_true = rel_obs_fun.observation_likelihood_array(x_test) + if calc_relative_error: + p_rel_y_true = rel_obs_fun.observation_likelihood_array(x_test) # Initial data x_rel, uvi_rel, uv_rel, y_rel, fuv_rel = rel_obs_fun.generate_n_observations(n_rel_train, n_xdim=1) @@ -134,13 +138,17 @@ def wrms2(y_true, y_est): y_abs_est = learner.model.abs_posterior_mean(x_test, fhat, vhat) best_points_est = set(np.argpartition(y_abs_est.flatten(), -n_best_points)[-n_best_points:]) - wrms_results[nl, 0, trial_number] = wrms(y_abs_true, y_abs_est) + wrms_results[nl, 0, trial_number] = test_data.wrms(y_abs_true, y_abs_est) true_pos_results[nl, 0, trial_number] = len(best_points_set.intersection(best_points_est)) - selected_error[nl, 0, trial_number] = wrms(y_abs_true[best_points], y_abs_est[best_points], weight=False) + selected_error[nl, 0, trial_number] = test_data.wrms(y_abs_true[best_points], y_abs_est[best_points], weight=False) + if calc_relative_error: + p_rel_y_post = learner.model.rel_posterior_likelihood_array(fhat=fhat, varhat=vhat) + relative_error[nl, 0, trial_number] = test_data.rel_error(y_abs_true, p_rel_y_true, y_abs_est, p_rel_y_post, weight=True) for obs_num in range(n_queries): - # learners[-2].obs_arguments['p_rel'] = max(0.0, (n_queries-obs_num)/float(n_queries)) + t0 = time.time() learners[-1].obs_arguments['p_rel'] = max(0.0, (n_queries-obs_num)/float(n_queries)) + for nl, learner in enumerate(learners): next_x = learner.model.select_observation(**learner.obs_arguments) if next_x.shape[0] == 1: @@ -159,43 +167,43 @@ def wrms2(y_true, y_est): # Get selected best point set and error results best_points_est = set(np.argpartition(y_abs_est.flatten(), -n_best_points)[-n_best_points:]) - wrms_results[nl, obs_num+1, trial_number] = wrms2(y_abs_true, y_abs_est) + wrms_results[nl, obs_num+1, trial_number] = test_data.wrms_misclass(y_abs_true, y_abs_est) true_pos_results[nl, obs_num+1, trial_number] = len(best_points_set.intersection(best_points_est)) - selected_error[nl, obs_num+1, trial_number] = wrms(y_abs_true[best_points], y_abs_est[best_points], weight=False) - - - print true_pos_results[:, obs_num+1, trial_number] - print wrms_results[:, obs_num+1, trial_number] + selected_error[nl, obs_num+1, trial_number] = test_data.wrms(y_abs_true[best_points], y_abs_est[best_points], weight=False) + if calc_relative_error: + p_rel_y_post = learner.model.rel_posterior_likelihood_array(fhat=fhat, varhat=vhat) + relative_error[nl, obs_num+1, trial_number] = test_data.rel_error(y_abs_true, p_rel_y_true, y_abs_est, + p_rel_y_post, weight=True) + if calc_relative_error: + print "{0}, t={t:0.2f}s, tp = {1}, wrms = {2}, p_err={3}".format(obs_num, true_pos_results[:, obs_num+1, trial_number], wrms_results[:, obs_num+1, trial_number], relative_error[:, obs_num+1, trial_number], t=time.time()-t0) + else: + print "{0}, t={t:0.2f}s, tp = {1}, wrms = {2}".format(obs_num, true_pos_results[:, obs_num+1, trial_number], wrms_results[:, obs_num+1, trial_number], t=time.time()-t0) for nl, learner in enumerate(learners): obs_tuple = learner.model.get_observations() obs_array[nl]['obs'].append(test_data.ObsObject(*obs_tuple)) - trial_number += 1 - except RuntimeError: - if randomize_waves == True: - print "Caught a bad laplace, try new trial wave" - continue - else: - raise - except np.linalg.LinAlgError: - if randomize_waves == True: - print "Caught a bad linalg inversion, try new trial wave" - continue - else: - raise -with open(data_dir+'wrms.pkl', 'wb') as fh: - pickle.dump(wrms_results, fh) + except Exception as e: + print 'Exception error is: %s, attempting a new wave' % e + continue + + trial_number += 1 + with open(data_dir+'wrms.pkl', 'wb') as fh: + pickle.dump(wrms_results[:,:,:trial_number], fh) + + with open(data_dir+'true_pos.pkl', 'wb') as fh: + pickle.dump(true_pos_results[:,:,:trial_number], fh) -with open(data_dir+'true_pos.pkl', 'wb') as fh: - pickle.dump(true_pos_results, fh) + with open(data_dir+'selected_error.pkl', 'wb') as fh: + pickle.dump(selected_error[:,:,:trial_number], fh) -with open(data_dir+'selected_error.pkl', 'wb') as fh: - pickle.dump(selected_error, fh) + if calc_relative_error: + with open(data_dir + 'relative_error.pkl', 'wb') as fh: + pickle.dump(relative_error[:, :, :trial_number], fh) -with open(data_dir+'obs.pkl', 'wb') as fh: - pickle.dump(obs_array, fh) + with open(data_dir+'obs.pkl', 'wb') as fh: + pickle.dump(obs_array, fh) -waver.save(data_dir+'wave_data.pkl') + waver.save(data_dir+'wave_data.pkl') -hfig = plot_statruns.plot_results(wrms_results, true_pos_results, selected_error, obs_array, data_dir=data_dir, bars=True, norm_comparator=0) \ No newline at end of file +hfig = plot_statruns.plot_results(wrms_results, true_pos_results, selected_error, obs_array, relative_error=relative_error, data_dir=data_dir, bars=True, norm_comparator=0) \ No newline at end of file diff --git a/plot_statruns.py b/plot_statruns.py index f04ef01..9f73230 100644 --- a/plot_statruns.py +++ b/plot_statruns.py @@ -18,12 +18,12 @@ def single_plot(data, x=None, names=None, title='', xlabel='Number of samples', hf, hax = plt.subplots() hl = [] for dd in data: - mean_err = np.mean(dd, axis=1) + mean_err = np.nanmean(dd, axis=1) if percentile < 0: - err_lo = np.std(dd, axis=1, ddof=1) / np.sqrt(n_trials) + err_lo = np.nanstd(dd, axis=1, ddof=1) / np.sqrt(n_trials) err_hi = err_lo elif percentile == 0: - err_lo = np.std(dd, axis=1) + err_lo = np.nanstd(dd, axis=1) err_hi = err_lo else: err_lo = mean_err - np.percentile(dd, percentile, axis=1) @@ -40,8 +40,7 @@ def single_plot(data, x=None, names=None, title='', xlabel='Number of samples', hax.set_ylabel(ylabel) return hf, hax - -def plot_results(wrms_results, true_pos_results, selected_error, obs_array, data_dir=None, bars=True, norm_comparator=0, exclusions=[4]): +def plot_results(wrms_results, true_pos_results, selected_error, obs_array, relative_error=None, data_dir=None, bars=True, norm_comparator=0, exclusions=[]): methods_indexes = [] for i in range(wrms_results.shape[0]): if i not in exclusions: @@ -54,6 +53,7 @@ def plot_results(wrms_results, true_pos_results, selected_error, obs_array, data true_pos_results=true_pos_results[methods_indexes,:,:] selected_error=selected_error[methods_indexes,:,:] + f0, ax0 = single_plot(wrms_results, names=names, ylabel='Weighted RMSE', bars=bars) f1, ax1 = single_plot(true_pos_results, names=names, ylabel='True positive selections (out of 15)', bars=True, precut=1, percentile=0) f2, ax2 = single_plot(selected_error, names=names, ylabel='RMSE of best paths', bars=True, precut=1) @@ -68,6 +68,13 @@ def plot_results(wrms_results, true_pos_results, selected_error, obs_array, data except: pass + if relative_error is not None: + relative_error=relative_error[methods_indexes,:,:] + fr, axr = single_plot(relative_error, names=names, ylabel='Mean relative prediction error', bars=True) + f.append(fr); ax.append(axr) + if data_dir is not None: + fr.savefig(data_dir + '/rel_error.pdf', bbox_inches='tight') + if data_dir is not None: f0.savefig(data_dir + '/wrms.pdf', bbox_inches='tight') f1.savefig(data_dir + '/true_pos.pdf', bbox_inches='tight') @@ -99,10 +106,16 @@ def load_and_plot(save_plots=True, *args, **kwargs): data_dir = askdirectory(initialdir='./data/') # open folder GUI wrms_results, true_pos_results, selected_error, obs_array = load_data(data_dir) + try: + with open(data_dir + '/relative_error.pkl', 'rb') as fh: + relative_error = pickle.load(fh) + except IOError: + print "No relative error data found." + relative_error = None if not save_plots: data_dir = None - hf = plot_results(wrms_results, true_pos_results, selected_error, obs_array, data_dir=data_dir, *args, **kwargs) + hf = plot_results(wrms_results, true_pos_results, selected_error, obs_array, data_dir=data_dir, relative_error=relative_error, *args, **kwargs) plt.show() return hf diff --git a/plot_tools.py b/plot_tools.py index 9b38bb7..3a0e662 100644 --- a/plot_tools.py +++ b/plot_tools.py @@ -5,7 +5,8 @@ from nice_plot_colors import * from cycler import cycler -plt.rc('axes', prop_cycle=(cycler('color', [greyify(c, .5, .8) for c in reversed(lines)]))) +# plt.rc('axes', prop_cycle=(cycler('color', [greyify(c, .5, .8) for c in reversed(lines)]))) +plt.rc('axes', prop_cycle=(cycler('color', lines))) def make_poly_array(x,y,sigma): nx = len(x) diff --git a/pref_active_learning.py b/pref_active_learning.py index b3cc99d..417f62a 100644 --- a/pref_active_learning.py +++ b/pref_active_learning.py @@ -6,41 +6,40 @@ import plot_tools as ptt import active_learners import test_data +import yaml from matplotlib.backends.backend_pdf import PdfPages nowstr = time.strftime("%Y_%m_%d-%H_%M") plt.rc('font',**{'family':'serif','sans-serif':['Computer Modern Roman']}) plt.rc('text', usetex=True) -save_plots = True +save_plots = False -# log_hyp = np.log([0.1,0.5,0.1,10.0]) # length_scale/s, sigma_f, sigma_n_rel, sigma_beta, v_beta -# log_hyp = np.log([0.07, 0.75, 0.25, 1.0, 28.1]) -# log_hyp = np.log([0.05, 1.5, 0.09, 2.0, 50.0]) -log_hyp = np.log([0.02, 0.5, 0.1, 0.8, 60.0]) -np.random.seed(2) +with open('./data/mid_freq_5.yaml', 'rt') as fh: + wave = yaml.safe_load(fh) -n_rel_train = 2 -n_abs_train = 1 -rel_sigma = 0.05 -delta_f = 1e-5 +try: + np.random.seed(wave['statrun_params']['randseed']) +except KeyError: + np.random.seed(0) + +n_rel_train = 5 +n_abs_train = 5 +n_queries = 1 -beta_sigma = 0.8 -beta_v = 80.0 +delta_f = 1e-5 n_xplot = 101 n_mcsamples = 1000 n_ysamples = 101 -n_queries = 30 +keep_f = True +verbose = 1 + +log_hyp = np.log(wave['hyperparameters']) # Define polynomial function to be modelled -# true_function = test_data.multi_peak -# random_wave = test_data.VariableWave([0.6, 1.0], [5.0, 10.0], [0.0, 1.0], [10.0, 20.0]) -random_wave = test_data.MultiWave(amp_range=[0.6, 1.2], f_range=[10.0, 30.0], off_range=[0.1, 0.9], - damp_range=[250.0, 350.0], n_components=3) -random_wave.randomize() -# random_wave.set_values(1.0, 6.00, 0.2, 10.50) +random_wave = test_data.MultiWave(**wave['wave_params']) true_function = random_wave.out random_wave.print_values() @@ -49,10 +48,13 @@ fig_dir = 'fig/' + nowstr + '/' ptt.ensure_dir(fig_dir) print "Figures will be saved to: {0}".format(fig_dir) + with open(fig_dir+'params.yaml', 'wt') as fh: + yaml.safe_dump(wave, fh) pdf_pages = PdfPages(fig_dir+'posterior_all.pdf') -rel_obs_fun = GPpref.RelObservationSampler(true_function, GPpref.PrefProbit(sigma=rel_sigma)) -abs_obs_fun = GPpref.AbsObservationSampler(true_function, GPpref.AbsBoundProbit(sigma=beta_sigma, v=beta_v)) +rel_obs_fun = GPpref.RelObservationSampler(true_function, GPpref.PrefProbit(**wave['rel_obs_params'])) +abs_obs_fun = GPpref.AbsObservationSampler(true_function, GPpref.AbsBoundProbit(**wave['abs_obs_params'])) +rel_sigma = wave['rel_obs_params']['sigma'] # True function x_plot = np.linspace(0.0,1.0,n_xplot,dtype='float') @@ -88,36 +90,43 @@ # learner = active_learners.ExpectedImprovementRel(**GP_kwargs) # obs_arguments = {'n_test': 100, 'zeta': 0.1, 'p_rel':1.0} -learner = active_learners.SampledClassification(**GP_kwargs) -obs_arguments = {'n_test':50, 'n_samples':10, 'y_threshold':0.8, 'p_pref_tol':1e-3, 'n_mc_abs':5} +# learner = active_learners.SampledClassification(verbose=verbose, **GP_kwargs) +# obs_arguments = {'n_test':50, 'n_samples':20, 'y_threshold':0.7, 'p_pref_tol':1e-3, 'n_mc_abs':50} + +learner = active_learners.DetSelect(**GP_kwargs) +obs_arguments = {'n_test': 100, 'n_rel_samples': 5, 'gamma': 2.0, 'tau': 0.5} + +# learner = active_learners.ActiveLearner(**GP_kwargs) +# obs_arguments = {'p_rel':0.0, 'n_rel_samples': 5} # Get initial solution learner.set_hyperparameters(log_hyp) f = learner.solve_laplace() learner.print_hyperparameters() +fig_p, (ax_p_l, ax_p_a, ax_p_r) = learner.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, + abs_y_samples, mc_samples) if save_plots: - fig_p, (ax_p_l, ax_p_a, ax_p_r) = learner.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, - abs_y_samples, mc_samples) pdf_pages.savefig(fig_p, bbox_inches='tight') # fig_p.savefig(fig_dir+'posterior00.pdf', bbox_inches='tight') for obs_num in range(n_queries): - if 'p_rel' in obs_arguments: - obs_arguments['p_rel'] = max(0.0, (n_queries - obs_num) / float(n_queries)) + # if 'p_rel' in obs_arguments: + # obs_arguments['p_rel'] = max(0.0, (n_queries - obs_num) / float(n_queries)) next_x = learner.select_observation(**obs_arguments) if next_x.shape[0] == 1: next_y, next_f = abs_obs_fun.generate_observations(next_x) - learner.add_observations(next_x, next_y) + learner.add_observations(next_x, next_y, keep_f=keep_f) print 'Abs: x:{0}, y:{1}'.format(next_x[0], next_y[0]) else: next_y, next_uvi, next_fx = rel_obs_fun.gaussian_multi_pairwise_sampler(next_x) next_fuv = next_fx[next_uvi][:, :, 0] fuv_rel = np.concatenate((fuv_rel, next_fuv), 0) - learner.add_observations(next_x, next_y, next_uvi) + learner.add_observations(next_x, next_y, next_uvi, keep_f=keep_f) print 'Rel: x:{0}, best_index:{1}'.format(next_x.flatten(), next_uvi[0, 1]) f = learner.solve_laplace() + if save_plots: fig_p, (ax_p_l, ax_p_a, ax_p_r) = learner.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, abs_y_samples, mc_samples) @@ -127,11 +136,11 @@ learner.print_hyperparameters() -if not save_plots: +if save_plots: + pdf_pages.close() +else: fig_p, (ax_p_l, ax_p_a, ax_p_r) = learner.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, abs_y_samples, mc_samples) -else: - pdf_pages.close() plt.show() print "Finished!" \ No newline at end of file diff --git a/test_data.py b/test_data.py index fc90a17..add42e3 100644 --- a/test_data.py +++ b/test_data.py @@ -2,10 +2,51 @@ import pickle +def wrms(y_true, y_est, weight=True): + # RMS weighted by the true value of y (high value high importance) + if weight: + w = y_true + else: + w = 1.0 + return np.sqrt(np.mean(((y_true - y_est)*w)**2)) + + +def wrms_misclass(y_true, y_est): + # This is the misclassification error, where the weight is the max of the true or predicted value (penalise + # predicting high values if the true value is low) + w = np.power(np.maximum(y_true, y_est), 2) + return np.sqrt(np.mean(((y_true - y_est)*w)**2)) + + +def rel_error(y_true, prel_true, y_est, prel_est, weight=False): + if weight: + y_max = np.maximum(y_true.flatten(), y_est.flatten()) + else: + y_max = np.ones(y_true.shape[0], dtype='float') + nx = prel_true.shape[0] + mean_p_err = 0.0 + w_sum = 0.0 + for i in range(nx): + for j in range(i, nx): + w = max(y_max[i], y_max[j]) + w_sum += w + mean_p_err += w*np.abs(prel_true[i,j] - prel_est[i,j]) + return mean_p_err/w_sum + + class ObsObject(object): def __init__(self, x_rel, uvi_rel, x_abs, y_rel, y_abs): self.x_rel, self.uvi_rel, self.x_abs, self.y_rel, self.y_abs = x_rel, uvi_rel, x_abs, y_rel, y_abs +def obs_stats(obs_array, n_rel_samples): + for method in obs_array: + n_rel = 0.0 + n_abs = 0.0 + for ob in method['obs']: + n_rel += ob.x_rel.shape[0]/n_rel_samples + n_abs += ob.x_abs.shape[0] + print "{0}: p_rel = {1}, p_abs = {2}".format(method['name'], n_rel/(n_rel+n_abs), n_abs/(n_rel+n_abs)) + class VariableWave(object): def __init__(self, amp_range, f_range, off_range, damp_range, n_components=1): @@ -89,8 +130,10 @@ def __init__(self, n_trials, n_components): self.frequency = np.zeros((n_trials, n_components), dtype='float') self.offset = np.zeros((n_trials, n_components), dtype='float') self.damping = np.zeros((n_trials, n_components), dtype='float') + self.n = 0 def set_vals(self, n, a, f, o, d): + self.n = n self.amplitude[n] = a self.frequency[n] = f self.offset[n] = o @@ -128,3 +171,37 @@ def data1(): mu_abs = np.array([[0.0]]) return x_rel, uvi_rel, uv_rel, y_rel, fuv_rel, x_abs, y_abs, mu_abs + + +def data2(): + x_rel = np.array([[ 0.8517731 ], [ 0.66358258], + [ 0.06054717], [ 0.45331369], + [ 0.8461625 ], [ 0.58854979]]) + uvi_rel = np.array([[0, 1], [2, 3], [4, 5]], dtype='int') + uv_rel = x_rel[uvi_rel][:,:,0] + y_rel = np.array([[-1], [1], [1]], dtype='int') + fuv_rel = np.array([[0.0043639, -0.10653237], [0.01463141, 0.05046293], + [0.01773679, 0.45730181]]) + + x_abs = np.array([[0.43432351]]) + y_abs = np.array([[0.38966307]]) + mu_abs = np.array([[0.0]]) + + return x_rel, uvi_rel, uv_rel, y_rel, fuv_rel, x_abs, y_abs, mu_abs + + +def data3(): + x_rel = np.array([[0.8517731 ], [0.66358258], + [0.06054717], [0.45331369], + [0.8461625 ], [0.58854979]]) + uvi_rel = np.array([[0, 1], [2, 3], [4, 5]], dtype='int') + uv_rel = x_rel[uvi_rel][:,:,0] + y_rel = np.array([[-1], [1], [1]], dtype='int') + fuv_rel = np.array([[0.0043639, -0.10653237], [0.01463141, 0.05046293], + [0.01773679, 0.45730181]]) + + x_abs = np.array([[0.43432351], [0.03362113]]) + y_abs = np.array([[0.38966307], [0.999]]) + mu_abs = np.array([[0.0], [0.0]]) + + return x_rel, uvi_rel, uv_rel, y_rel, fuv_rel, x_abs, y_abs, mu_abs From b7cf27f95c184ba7a0bd17550fec34324f8b27ba Mon Sep 17 00:00:00 2001 From: Nicholas Lawrance Date: Wed, 1 Nov 2017 17:44:52 -0700 Subject: [PATCH 17/38] Added multi-directory plotting, messing with MaxVar --- active_learners.py | 36 ++++++++++++-- active_statruns.py | 99 +++++++++++++++---------------------- data/statruns2_oct2017.yaml | 37 ++++++++++++++ plot_statruns.py | 52 +++++++++++++++---- pref_active_learning.py | 13 +++-- 5 files changed, 160 insertions(+), 77 deletions(-) create mode 100644 data/statruns2_oct2017.yaml diff --git a/active_learners.py b/active_learners.py index f91c6e3..105184c 100644 --- a/active_learners.py +++ b/active_learners.py @@ -78,11 +78,41 @@ def create_posterior_plot(self, x_test, f_true, mu_true, rel_sigma, fuv_train, a class MaxVar(ActiveLearner): # Max variance - def select_observation(self, domain=None, n_test=100): + def select_observation(self, domain=None, n_test=100, n_rel_samples = 2, p_rel = 0.0, rel_tau = 1.0, abs_tau = 1.0e-5,w_v=1.0): + # If p_rel < 0 then we will select based on variance magnitude + if p_rel < 0.0: + p_select = 1.0 + else: + p_select = p_rel x_test = self.uniform_domain_sampler(n_test, domain) fhat, vhat = self.predict_latent(x_test) - return x_test[[np.argmax(np.diagonal(vhat))], :] + vv = np.sqrt(np.diagonal(vhat)) + if np.random.uniform() < p_select: # i.e choose a relative sample + available_indexes = set(range(len(x_test))) + dK = np.zeros(len(available_indexes)) + best_n = [softmax_selector(vv, tau=rel_tau)] + while len(best_n) < n_rel_samples: + dK[best_n[-1]] = -1e5 # Bit of a scam because it is still possible to sample this value + available_indexes.remove(best_n[-1]) + best_n.append(-1) + for cn in available_indexes: + best_n[-1] = cn + K = vhat[np.ix_(best_n, best_n)] + dK[cn] = np.linalg.det(K) + best_n[-1] = softmax_selector(dK, tau=rel_tau) # np.argmax(dK) # + else: + best_n = [softmax_selector(vv, tau=abs_tau)] #[np.argmax(ucb)] # + + # This chooses an absolute query based on determinant + if p_rel < 0.0: + best_detK = -p_rel*np.sqrt(dK[best_n[-1]]) + p_select = best_detK/(best_detK + vv.max()) + # print "p_select: {0}".format(p_select) + if np.random.uniform() > p_select: + # if p_select < 0.5: + best_n = [softmax_selector(w_v*vv+(1.0-w_v)*fhat.flatten(), tau=abs_tau)] + return x_test[best_n, :] class UCBLatent(ActiveLearner): # All absolute returns @@ -206,7 +236,7 @@ def select_observation(self, domain=None, n_test=100, p_rel=0.5, n_rel_samples=2 while len(best_n) < n_rel_samples: # ucb = ucb*sq_dist[best_n[-1], :] # Discount ucb by distance ucb[best_n[-1]] = 0.0 - ucb *= 4*p_rel_y[best_n[-1],:]*(1.0 - p_rel_y[best_n[-1],:]) # Divide by likelihood that each point is better than previous best + ucb *= 4*p_rel_y[best_n[-1],:]*(1.0 - p_rel_y[best_n[-1],:]) # Reduce by likelihood that each point is better than previous best best_n.append(softmax_selector(ucb, tau=tau)) # best_n.append(np.argmax(ucb)) else: diff --git a/active_statruns.py b/active_statruns.py index 6985f9f..fdc2dc0 100644 --- a/active_statruns.py +++ b/active_statruns.py @@ -12,10 +12,11 @@ np.set_printoptions(precision=3) class Learner(object): - def __init__(self, model_type, obs_arguments, name): - self.model_type = getattr(active_learners, self.model_type) - self.obs_arguments = obs_arguments + def __init__(self, model_type, obs_args, name, update_p_rel = False): + self.model_type = getattr(active_learners, model_type) + self.obs_arguments = obs_args self.name = name + self.update_p_rel = update_p_rel def build_model(self, training_data): self.model = self.model_type(**training_data) @@ -23,75 +24,49 @@ def build_model(self, training_data): now_time = time.strftime("%Y_%m_%d-%H_%M") with open('./data/statruns2_oct2017.yaml', 'rt') as fh: - statrun_params = yaml.safe_load(fh) + run_parameters = yaml.safe_load(fh) -log_hyp = np.log(statrun_params['hyperparameters']) +log_hyp = np.log(run_parameters['hyperparameters']) # Statrun parameters -np.random.seed(statrun_params['statrun_params']['randseed']) -n_rel_train = statrun_params['statrun_params']['n_rel_train'] -n_abs_train = statrun_params['statrun_params']['n_abs_train'] -n_xtest = statrun_params['statrun_params']['n_xtest'] -n_best_points = statrun_params['statrun_params']['n_best_points'] -n_trials = statrun_params['statrun_params']['n_trials'] -n_queries = statrun_params['statrun_params']['n_queries'] -if 'calc_relative_error' in statrun_params['statrun_params']: - calc_relative_error = statrun_params['statrun_params']['calc_relative_error'] +statrun_params = run_parameters['statrun_params'] +np.random.seed(statrun_params['randseed']) +n_best_points = statrun_params['n_best_points'] +n_trials = statrun_params['n_trials'] +n_queries = statrun_params['n_queries'] +if 'calc_relative_error' in run_parameters['statrun_params']: + calc_relative_error = statrun_params['calc_relative_error'] else: calc_relative_error = False # Learner parameters -n_rel_samples = statrun_params['learner_params']['n_rel_samples'] -delta_f = statrun_params['learner_params']['delta_f'] - +n_rel_samples = run_parameters['learner_params']['n_rel_samples'] # Define polynomial function to be modelled -random_wave = test_data.MultiWave(**statrun_params['wave_params']) +random_wave = test_data.MultiWave(**run_parameters['wave_params']) now_time = time.strftime("%Y_%m_%d-%H_%M") data_dir = 'data/' + now_time + '/' ptt.ensure_dir(data_dir) print "Data will be saved to: {0}".format(data_dir) waver = test_data.WaveSaver(n_trials, random_wave.n_components) -with open(data_dir + 'params.yaml', 'wt') as fh: - yaml.safe_dump(statrun_params, fh) + # True function -x_plot = np.linspace(0.0, 1.0, n_xtest,dtype='float') +x_plot = np.linspace(0.0, 1.0, statrun_params['n_xtest'], dtype='float') x_test = np.atleast_2d(x_plot).T -# Learners +# Construct active learner objects +n_learners = len(run_parameters['learners']) learners = [] -for l in statrun_params['learners']: +names = [] +obs_array = [] +for l in run_parameters['learners']: + if 'n_rel_samples' in l['obs_args']: + l['obs_args']['n_rel_samples'] = n_rel_samples learners.append(Learner(**l)) - -# Construct active learner object -learners = [Learner(active_learners.ActiveLearner, {'p_rel': 1.0, 'n_rel_samples': n_rel_samples}, 'Random (rel)'), - Learner(active_learners.ActiveLearner, {'p_rel': 0.0, 'n_rel_samples': n_rel_samples}, 'Random (abs)'), - Learner(active_learners.ActiveLearner, {'p_rel': 0.5, 'n_rel_samples': n_rel_samples}, 'Random ($p_{rel}=0.5$)'), - Learner(active_learners.MaxVar, {'n_test': 100}, 'MaxVar (abs)'), - Learner(active_learners.UCBLatent, {'gamma': 3.0, 'n_test': 100}, 'UCB (abs)'), - Learner(active_learners.UCBLatentSoftmax, {'gamma': 2.0, 'n_test': 100, 'tau':1.0}, 'UCBSoft (abs)'), - Learner(active_learners.UCBCovarianceSoftmax, {'gamma': 2.0, 'n_test': 100}, 'UCBCovSoft (abs)'), - Learner(active_learners.DetRelBoo, {'n_test': 100, 'n_rel_samples': n_rel_samples, 'gamma': 2.0, 'tau': 0.5}, 'DetRelBoo (rel)'), - Learner(active_learners.DetSelect, {'n_test': 100, 'n_rel_samples': n_rel_samples, 'gamma': 2.0, 'abs_tau': -1, 'rel_tau': 1.0e-5}, 'DetSelectRel (rel)'), - Learner(active_learners.DetSelect, {'n_test': 100, 'n_rel_samples': n_rel_samples, 'gamma': 2.0, 'abs_tau': 1.0e-5, 'rel_tau': 1.0e-5}, 'DetSelectGreedy (rel, abs)'), - Learner(active_learners.DetSelect, {'n_test': 100, 'n_rel_samples': n_rel_samples, 'gamma': 2.0, 'abs_tau': 0.5, 'rel_tau': 0.5}, 'DetSelectSoft (rel, abs)'), - Learner(active_learners.ExpectedImprovementAbs, { 'n_test': 100 }, 'EI (Abs)'), - Learner(active_learners.ExpectedImprovementRel, { 'n_test': 100 }, 'EI (Rel)'), - Learner(active_learners.UCBAbsRel, { 'n_test': 100, 'n_rel_samples': n_rel_samples, 'gamma': 2.0, 'tau': 1.0}, 'UCBCombined (rel, abs)'), - # Learner(active_learners.UCBAbsRelD, { 'n_test': 100, 'n_rel_samples': n_rel_samples, 'gamma': 2.0, 'tau': 1.0}, 'UCBD (rel and abs)'), - # Learner(active_learners.ABSThresh, {'n_test': 100, 'p_thresh': 0.7}, 'ABSThresh'), - # Learner(active_learners.PeakComparitor, {'gamma': 2.0, 'n_test': 50, 'n_rel_samples': n_rel_samples}, 'PeakComparitor'), - # Learner(active_learners.LikelihoodImprovement, {'req_improvement': 0.60, 'n_test': 50, 'gamma': 2.0, 'n_rel_samples': n_rel_samples, 'p_thresh': 0.7}, 'LikelihoodImprovement'), - # Learner(active_learners.SampledThreshold, {'n_test':50, 'n_samples':10, 'y_threshold':0.8, 'p_pref_tol':1e-3, 'n_mc_abs':5}, 'SampledThreshold'), - # Learner(active_learners.SampledClassification, {'n_test':50, 'n_samples':10, 'y_threshold':0.8, 'p_pref_tol':1e-3, 'n_mc_abs':5}, 'SampledClassification'), - ] -names = [l.name for l in learners] -assert len(names) == len(learners), "Number of names does not match number of learners." -n_learners = len(learners) - -obs_array = [{'name': name, 'obs': []} for name in names] + names.append(l['name']) + obs_array.append({'name': l['name'], 'obs': []}) wrms_results = np.zeros((n_learners, n_queries+1, n_trials)) true_pos_results = np.zeros((n_learners, n_queries+1, n_trials), dtype='int') @@ -101,8 +76,10 @@ def build_model(self, training_data): else: relative_error = None +with open(data_dir + 'params.yaml', 'wt') as fh: + yaml.safe_dump(run_parameters, fh) + trial_number = 0 -# for trial_number in range(n_trials): while trial_number < n_trials: try: @@ -110,23 +87,21 @@ def build_model(self, training_data): random_wave.randomize() random_wave.print_values() waver.set_vals(trial_number, *random_wave.get_values()) - rel_obs_fun = GPpref.RelObservationSampler(random_wave.out, GPpref.PrefProbit(**statrun_params['rel_obs_params'])) - abs_obs_fun = GPpref.AbsObservationSampler(random_wave.out, GPpref.AbsBoundProbit(**statrun_params['abs_obs_params'])) + rel_obs_fun = GPpref.RelObservationSampler(random_wave.out, GPpref.PrefProbit(**run_parameters['rel_obs_params'])) + abs_obs_fun = GPpref.AbsObservationSampler(random_wave.out, GPpref.AbsBoundProbit(**run_parameters['abs_obs_params'])) f_true = abs_obs_fun.f(x_test) y_abs_true = abs_obs_fun.mean_link(x_test) best_points = np.argpartition(y_abs_true.flatten(), -n_best_points)[-n_best_points:] best_points_set = set(best_points) - # abs_y_samples = np.atleast_2d(np.linspace(0.01, 0.99, n_ysamples)).T - # p_abs_y_true = abs_obs_fun.observation_likelihood_array(x_test, abs_y_samples) if calc_relative_error: p_rel_y_true = rel_obs_fun.observation_likelihood_array(x_test) # Initial data - x_rel, uvi_rel, uv_rel, y_rel, fuv_rel = rel_obs_fun.generate_n_observations(n_rel_train, n_xdim=1) - x_abs, y_abs, mu_abs = abs_obs_fun.generate_n_observations(n_abs_train, n_xdim=1) + x_rel, uvi_rel, uv_rel, y_rel, fuv_rel = rel_obs_fun.generate_n_observations(statrun_params['n_rel_train'], n_xdim=1) + x_abs, y_abs, mu_abs = abs_obs_fun.generate_n_observations(statrun_params['n_abs_train'], n_xdim=1) training_data = {'x_rel': x_rel, 'uvi_rel': uvi_rel, 'x_abs': x_abs, 'y_rel': y_rel, 'y_abs': y_abs, - 'delta_f': delta_f, 'rel_likelihood': GPpref.PrefProbit(), + 'delta_f': run_parameters['learner_params']['delta_f'], 'rel_likelihood': GPpref.PrefProbit(), 'abs_likelihood': GPpref.AbsBoundProbit()} # Get initial solution @@ -147,9 +122,13 @@ def build_model(self, training_data): for obs_num in range(n_queries): t0 = time.time() - learners[-1].obs_arguments['p_rel'] = max(0.0, (n_queries-obs_num)/float(n_queries)) + linear_p_rel = max(0.0, (n_queries-obs_num)/float(n_queries)) for nl, learner in enumerate(learners): + # Update p_rel for certain methods (hacky?) + if learner.update_p_rel: + learner.obs_arguments['p_rel'] = linear_p_rel + next_x = learner.model.select_observation(**learner.obs_arguments) if next_x.shape[0] == 1: next_y, next_f = abs_obs_fun.generate_observations(next_x) diff --git a/data/statruns2_oct2017.yaml b/data/statruns2_oct2017.yaml new file mode 100644 index 0000000..ba2de29 --- /dev/null +++ b/data/statruns2_oct2017.yaml @@ -0,0 +1,37 @@ +# Hyperparameters are: [l, sig_f, sig_rel, sig_beta, v_beta] +hyperparameters: [0.025, 1.0, 0.2, 1.0, 40.0] +abs_obs_params: {sigma: 1.0, v: 40.0} +rel_obs_params: {sigma: 0.2} +wave_params: + amp_range: [1.5, 1.8] + damp_range: [250.0, 350.0] + f_range: [10.0, 30.0] + n_components: 4 + off_range: [0.01, 0.99] +statrun_params: + n_queries: 80 + n_trials: 100 + n_rel_train: 0 + n_abs_train: 1 + randseed: 0 + n_best_points: 30 + n_xtest: 200 + calc_relative_error: True +learner_params: {delta_f: 1.0e-05, n_rel_samples: 5} +# High tau for softmax is more random +learners: +# - name: MaxVar (abs) +# model_type: MaxVar +# obs_args: {p_rel: 0.0} + - name: DetKSwitch ($p_{rel}=-5.0, \tau=0.1$) (rel, abs) + model_type: MaxVar + obs_args: {n_rel_samples: 5, p_rel: -5.0, rel_tau: 0.1, abs_tau: 0.1} + - name: DetKSwitch ($p_{rel}=-5.0, \tau=10^{-5}$) (rel, abs) + model_type: MaxVar + obs_args: {n_rel_samples: 5, p_rel: -5.0, rel_tau: 1e-5, abs_tau: 1e-5} + - name: DetKUCBHard ($w_v=0.75, \tau=10^{-5}$) (rel, abs) + model_type: MaxVar + obs_args: {n_rel_samples: 5, p_rel: -5.0, rel_tau: 1e-5, abs_tau: 1e-5, w_v: 0.75} + - name: DetKUCB ($w_v=0.75, \tau=0.1$) (rel, abs) + model_type: MaxVar + obs_args: {n_rel_samples: 5, p_rel: -5.0, rel_tau: 0.1, abs_tau: 0.1, w_v: 0.75} diff --git a/plot_statruns.py b/plot_statruns.py index 9f73230..3e64dac 100644 --- a/plot_statruns.py +++ b/plot_statruns.py @@ -86,7 +86,13 @@ def plot_results(wrms_results, true_pos_results, selected_error, obs_array, rela plt.show() return f -def load_data(data_dir): +def load_data(data_dir=None): + if data_dir is None: + Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing + data_dir = askdirectory(initialdir='./data/') + if data_dir == '': # Cancel clicked + raise IOError('No directory selected') + with open(data_dir+'/wrms.pkl', 'rb') as fh: wrms_results = pickle.load(fh) # Dimensions n_learners, n_queries+1, n_trials @@ -99,13 +105,6 @@ def load_data(data_dir): with open(data_dir+'/obs.pkl', 'rb') as fh: obs_array = pickle.load(fh) - return wrms_results, true_pos_results, selected_error, obs_array - -def load_and_plot(save_plots=True, *args, **kwargs): - Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing - data_dir = askdirectory(initialdir='./data/') # open folder GUI - - wrms_results, true_pos_results, selected_error, obs_array = load_data(data_dir) try: with open(data_dir + '/relative_error.pkl', 'rb') as fh: relative_error = pickle.load(fh) @@ -113,11 +112,46 @@ def load_and_plot(save_plots=True, *args, **kwargs): print "No relative error data found." relative_error = None - if not save_plots: + return wrms_results, true_pos_results, selected_error, obs_array, relative_error + + +def load_and_plot(save_plots=True, *args, **kwargs): + Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing + data_dir = askdirectory(initialdir='./data/') + if data_dir == '': return None # Cancel clicked + wrms_results, true_pos_results, selected_error, obs_array, relative_error = load_data(data_dir) + + if save_plots is False: data_dir = None hf = plot_results(wrms_results, true_pos_results, selected_error, obs_array, data_dir=data_dir, relative_error=relative_error, *args, **kwargs) plt.show() return hf +def get_selection(): + wrms, true_pos, sel_err, obs, rel_err = load_data() + print "The following models were found:" + for i, obs_item in enumerate(obs): + print "{0}: {1}".format(i, obs_item['name']) + dex = input('Select desired models by index as an array: ') + wrms, true_pos, sel_err = wrms[dex, :, :], true_pos[dex, :, :], sel_err[dex, :, :] + if rel_err is not None: + rel_err = rel_err[dex, :, :] + obs = [obs[i] for i in dex] + finished = input('Finished? (0/1)') + return wrms, true_pos, sel_err, obs, rel_err, finished + + +def load_multiple(*args, **kwargs): + wrms, true_pos, sel_err, obs, rel_err, finished = get_selection() + app = lambda a, b: np.append(a, b, axis=0) + while not finished: + wrms2, true_pos2, sel_err2, obs2, rel_err2, finished = get_selection() + wrms, true_pos, sel_err = app(wrms, wrms2), app(true_pos, true_pos2), app(sel_err, sel_err2) + if rel_err is not None: rel_err = app(rel_err, rel_err2) + obs.extend(obs2) + hf = plot_results(wrms, true_pos, sel_err, obs, relative_error=rel_err, *args, **kwargs) + plt.show() + return hf, wrms, true_pos, sel_err, obs, rel_err + if __name__ == "__main__": hf = load_and_plot(save_plots=False, bars=True) \ No newline at end of file diff --git a/pref_active_learning.py b/pref_active_learning.py index 417f62a..62c6383 100644 --- a/pref_active_learning.py +++ b/pref_active_learning.py @@ -23,9 +23,9 @@ except KeyError: np.random.seed(0) -n_rel_train = 5 -n_abs_train = 5 -n_queries = 1 +n_rel_train = 0 +n_abs_train = 1 +n_queries = 50 delta_f = 1e-5 @@ -93,12 +93,15 @@ # learner = active_learners.SampledClassification(verbose=verbose, **GP_kwargs) # obs_arguments = {'n_test':50, 'n_samples':20, 'y_threshold':0.7, 'p_pref_tol':1e-3, 'n_mc_abs':50} -learner = active_learners.DetSelect(**GP_kwargs) -obs_arguments = {'n_test': 100, 'n_rel_samples': 5, 'gamma': 2.0, 'tau': 0.5} +# learner = active_learners.DetSelect(**GP_kwargs) +# obs_arguments = {'n_test': 100, 'n_rel_samples': 5, 'gamma': 2.0, 'tau': 0.5} # learner = active_learners.ActiveLearner(**GP_kwargs) # obs_arguments = {'p_rel':0.0, 'n_rel_samples': 5} +learner = active_learners.MaxVar(**GP_kwargs) +obs_arguments = {'n_rel_samples': 5, 'p_rel': -1, 'rel_tau': 0.01, 'abs_tau': 0.01} + # Get initial solution learner.set_hyperparameters(log_hyp) f = learner.solve_laplace() From 3d41512cbcba1b4cea75ec7da8f18abfacb7e8e1 Mon Sep 17 00:00:00 2001 From: Nicholas Lawrance Date: Thu, 9 Nov 2017 18:02:13 -0800 Subject: [PATCH 18/38] Added OrdinalProbit likelihood class to GPpref and added a demonstration script ordinal_likelihoods.py --- GP_preference_demo.py | 5 +- GPpref.py | 102 ++++++++++++++++++++++++++++++++++++ active_learners.py | 24 ++++++--- active_statruns.py | 36 +++---------- beta_to_discrete.py | 61 +++++++++++++++++++++ data/statruns2_oct2017.yaml | 40 +++++++++++--- ordinal_likelihoods.py | 76 +++++++++++++++++++++++++++ plot_statruns.py | 85 +++++++++++++++++++----------- pref_active_learning.py | 71 +++++++++++++------------ 9 files changed, 389 insertions(+), 111 deletions(-) create mode 100644 beta_to_discrete.py create mode 100644 ordinal_likelihoods.py diff --git a/GP_preference_demo.py b/GP_preference_demo.py index afd44c4..47b8731 100644 --- a/GP_preference_demo.py +++ b/GP_preference_demo.py @@ -14,7 +14,7 @@ use_test_data = False # test_data.data3 # verbose = 2 -with open('./data/mid_freq_5.yaml', 'rt') as fh: +with open('./data/statruns_nov2017.yaml', 'rt') as fh: wave = yaml.safe_load(fh) try: @@ -25,7 +25,7 @@ log_hyp = np.log(wave['hyperparameters']) n_rel_train = 5 -n_abs_train = 5 +n_abs_train = 15 delta_f = 1e-5 @@ -40,7 +40,6 @@ rel_obs_fun = GPpref.RelObservationSampler(true_function, GPpref.PrefProbit(**wave['rel_obs_params'])) abs_obs_fun = GPpref.AbsObservationSampler(true_function, GPpref.AbsBoundProbit(**wave['abs_obs_params'])) -# Main program # True function x_plot = np.linspace(0.0,1.0,n_xplot,dtype='float') x_test = np.atleast_2d(x_plot).T diff --git a/GPpref.py b/GPpref.py index 0486b06..551fc51 100644 --- a/GPpref.py +++ b/GPpref.py @@ -150,6 +150,107 @@ def generate_samples(self, f): y[fuv[:, 1] > fuv[:, 0]] = 1 return y, fuv + +class OrdinalProbit(object): + def __init__(self, sigma=1.0, b=1.0, n_ordinals=5, eps=1.0e-10): + self.set_sigma(sigma) + self.set_b(b, n_ordinals) + self.eps = eps + + def set_b(self, b, n_ordinals): + self.n_ordinals = n_ordinals + if not hasattr(b, "__len__"): + b = abs(b) + self.b = np.hstack(([-np.Inf],np.linspace(-b, b, self.n_ordinals-1), [np.Inf])) + elif len(b) == self.n_ordinals+1: + self.b = b + elif len(b) == self.n_ordinals-1: + self.b = np.hstack(([-np.Inf], b, [np.Inf])) + else: + raise ValueError('Specified b should be a scalar or vector of breakpoints') + + def set_sigma(self, sigma): + self.sigma = sigma + self._isigma = 1.0/self.sigma + self._ivar = self._isigma**2 + + def print_hyperparameters(self): + print "Ordinal probit, {0} ordered categories.".format(self.n_ordinals), + print "Sigma: {0:0.2f}, b: ".format(self.sigma), + print self.b + + def z_k(self, y, f): + return self._isigma*(self.b[y] - f) + + def norm_pdf(self, y, f): + out = np.zeros(y.shape, dtype='float') + for i in range(out.shape[0]): + if y[i] != 0 and y[i] != self.n_ordinals: # b0 = -Inf -> N(-Inf) = 0 + z = self._isigma*(self.b[y[i]] - f[i]) + out[i] = std_norm_pdf(z) + return out + + def norm_cdf(self, y, f): + out = np.zeros(y.shape, dtype='float') + for i in range(out.shape[0]): + if y[i] == self.n_ordinals: + out[i] = 1.0 + elif y[i] != 0: + z = self._isigma*(self.b[y[i]] - f[i]) + out[i] = std_norm_cdf(z) + return out + + def z_pdf(self, y, f): + out = np.zeros(y.shape, dtype='float') + for i in range(out.shape[0]): + if y[i] != 0 and y[i] != self.n_ordinals: # b0 = -Inf -> N(-Inf) = 0 + z = self._isigma*(self.b[y[i]] - f[i]) + out[i] = z*std_norm_pdf(z) + return out + + + def likelihood(self, y, f): + return self.norm_cdf(y, f) - self.norm_cdf(y-1, f) + + def log_likelihood(self, y, f): + return np.log(self.likelihood(y, f)) + + def derivatives(self, y, f): + + l = self.likelihood(y, f) + py = np.log(l) + + # First derivative - Chu and Gharamani + # Having issues with derivative (likelihood denominator drops to 0) + dpy_df = np.zeros(l.shape, dtype='float') + d2py_df2 = np.zeros(l.shape, dtype='float') + for i in range(l.shape[0]): + if l[i] < self.eps: + # l2 = self.likelihood(y[i], f[i]+self.delta_f) + # l0 = self.likelihood(y[i], f[i]-self.delta_f) + # dpy_df[i] = -(l2-l[i])/self.delta_f/l[i] # (ln(f))' = f'/f + # d2py_df2[i] = (l2 - 2*l[i] + l0)/self.delta_f**2/dpy_df[i]/l[i] + + if y[i] == 1: + dpy_df[i] = -self._isigma*self.z_k(y[i], f[i]) + d2py_df2[i] = self._ivar + elif y[i] == self.n_ordinals: + dpy_df[i] = -self._isigma*self.z_k(y[i]-1, f[i]) + d2py_df2[i] = self._ivar + else: + z1 = self.z_k(y[i], f[i]) + z2 = self.z_k(y[i]-1, f[i]) + ep = np.exp(-0.5*(z1**2 - z2**2)) + dpy_df[i] = -self._isigma*(z1*ep-z2)/(ep - 1.0) + d2py_df2[i] = self._ivar*(1.0 - (z1**2 *ep - z2**2)/(ep - 1.0)) + dpy_df[i]**2 + else: + dpy_df[i] = self._isigma*(self.norm_pdf(y[i], f[i]) - self.norm_pdf(y[i]-1, f[i])) / l[i] + d2py_df2[i] = dpy_df[i]**2 + self._ivar*(self.z_pdf(y[i], f[i]) - self.z_pdf(y[i]-1, f[i])) / l[i] + + W = np.diagflat(d2py_df2) + return W, dpy_df, py + + class AbsBoundProbit(object): def __init__(self, sigma=1.0, v=10.0): # v is the precision, kind of related to inverse of noise, high v is sharp distributions @@ -203,6 +304,7 @@ def derivatives(self, y, f): aa, bb = self.get_alpha_beta(f) + # Trouble with derivatives... dpy_df = self.v*self._isqrt2sig*std_norm_pdf(f*self._isqrt2sig) * (np.log(y) - np.log(1-y) - digamma(aa) + digamma(bb)) Wdiag = - self.v*self._isqrt2sig*std_norm_pdf(f*self._isqrt2sig) * ( diff --git a/active_learners.py b/active_learners.py index 105184c..7aeb817 100644 --- a/active_learners.py +++ b/active_learners.py @@ -3,7 +3,7 @@ from scipy.stats import beta import plot_tools as ptt import time - +import sys def calc_ucb(fhat, vhat, gamma=2.0, sigma_offset=0.0): # return fhat + gamma * (np.sqrt(np.atleast_2d(vhat.diagonal()).T) - sigma_offset) @@ -16,6 +16,18 @@ def softmax_selector(x, tau=1.0): Px = ex/ex.sum() return np.random.choice(len(x), p=Px) +class Learner(object): + def __init__(self, model_type, obs_args, name, update_p_rel = False): + self.model_type = getattr(sys.modules[__name__], model_type) + self.obs_arguments = obs_args + self.name = name + self.update_p_rel = update_p_rel + + def build_model(self, training_data): + self.model = self.model_type(**training_data) + + def select_observation(self): + return self.model.select_observation(**self.obs_arguments) class ActiveLearner(GPpref.PreferenceGaussianProcess): def init_extras(self): @@ -104,13 +116,11 @@ def select_observation(self, domain=None, n_test=100, n_rel_samples = 2, p_rel = else: best_n = [softmax_selector(vv, tau=abs_tau)] #[np.argmax(ucb)] # - # This chooses an absolute query based on determinant - if p_rel < 0.0: + # This chooses an absolute query based on determinant. Choose relK using beta cdf likelihood + if p_rel <= -1.0: best_detK = -p_rel*np.sqrt(dK[best_n[-1]]) - p_select = best_detK/(best_detK + vv.max()) - # print "p_select: {0}".format(p_select) - if np.random.uniform() > p_select: - # if p_select < 0.5: + K_ratio = best_detK/(vv.max() + best_detK) + if np.random.uniform() > beta.cdf(K_ratio, -p_rel, -p_rel): best_n = [softmax_selector(w_v*vv+(1.0-w_v)*fhat.flatten(), tau=abs_tau)] return x_test[best_n, :] diff --git a/active_statruns.py b/active_statruns.py index fdc2dc0..e7b3622 100644 --- a/active_statruns.py +++ b/active_statruns.py @@ -5,25 +5,14 @@ import plot_tools as ptt import active_learners import test_data -import pickle import plot_statruns import yaml np.set_printoptions(precision=3) -class Learner(object): - def __init__(self, model_type, obs_args, name, update_p_rel = False): - self.model_type = getattr(active_learners, model_type) - self.obs_arguments = obs_args - self.name = name - self.update_p_rel = update_p_rel - - def build_model(self, training_data): - self.model = self.model_type(**training_data) - now_time = time.strftime("%Y_%m_%d-%H_%M") -with open('./data/statruns2_oct2017.yaml', 'rt') as fh: +with open('./data/statruns_nov2017.yaml', 'rt') as fh: run_parameters = yaml.safe_load(fh) log_hyp = np.log(run_parameters['hyperparameters']) @@ -64,7 +53,7 @@ def build_model(self, training_data): for l in run_parameters['learners']: if 'n_rel_samples' in l['obs_args']: l['obs_args']['n_rel_samples'] = n_rel_samples - learners.append(Learner(**l)) + learners.append(active_learners.Learner(**l)) names.append(l['name']) obs_array.append({'name': l['name'], 'obs': []}) @@ -167,21 +156,12 @@ def build_model(self, training_data): continue trial_number += 1 - with open(data_dir+'wrms.pkl', 'wb') as fh: - pickle.dump(wrms_results[:,:,:trial_number], fh) - - with open(data_dir+'true_pos.pkl', 'wb') as fh: - pickle.dump(true_pos_results[:,:,:trial_number], fh) - - with open(data_dir+'selected_error.pkl', 'wb') as fh: - pickle.dump(selected_error[:,:,:trial_number], fh) - - if calc_relative_error: - with open(data_dir + 'relative_error.pkl', 'wb') as fh: - pickle.dump(relative_error[:, :, :trial_number], fh) - - with open(data_dir+'obs.pkl', 'wb') as fh: - pickle.dump(obs_array, fh) + if relative_error is not None: + plot_statruns.save_data(data_dir, wrms_results[:,:,:trial_number], true_pos_results[:, :, :trial_number], + selected_error[:, :, :trial_number], obs_array, relative_error[:, :, :trial_number]) + else: + plot_statruns.save_data(data_dir, wrms_results[:,:,:trial_number], true_pos_results[:, :, :trial_number], + selected_error[:, :, :trial_number], obs_array) waver.save(data_dir+'wave_data.pkl') diff --git a/beta_to_discrete.py b/beta_to_discrete.py new file mode 100644 index 0000000..6f86c37 --- /dev/null +++ b/beta_to_discrete.py @@ -0,0 +1,61 @@ +import numpy as np +import matplotlib.pyplot as plt +from matplotlib.patches import Rectangle +import GPpref +import nice_plot_colors as npc + +def convert_to_discrete(b, f, n_cat=5): + P = np.zeros(n_cat+1, dtype='float') + for i in range(1, n_cat+1): + P[i] = b.cdf(float(i)/n_cat, f) - P[0:i].sum() + return P[1:] + +def draw_probs(ax, x, P): + del_x = x[1]-x[0] + ddel_x = del_x / P.shape[0] + for xi, p in zip(x, P.T): # Loop over columns + ip = np.argsort(p, ) + for i in ip[::-1]: + box = Rectangle((xi+ddel_x*i, 0.0), ddel_x, p[i], color=npc.bars[i]) + ax.add_artist(box) + +# Examine how continuous variable is converted to discrete categories through beta distribution +sigma = 1.0 +v = 10.0 +n = 5 + +beta_likelihood = GPpref.AbsBoundProbit(sigma=sigma, v=v) + +f = np.linspace(-3.0, 3.0, 7) + +y = np.linspace(0.0, 1.0, 201) +py = np.zeros((len(f), len(y)), dtype='float') +Py = np.zeros((len(f), n), dtype='float') +for i, fi in enumerate(f): + py[i] = beta_likelihood.likelihood(y, fi) + Py[i] = convert_to_discrete(beta_likelihood, fi, n) + +fh, ah = plt.subplots() +xbars = np.arange(0.0, 1.0, step=1.0/n) +draw_probs(ah, xbars, Py) +ah2 = ah.twinx() +lines = [] +for i, p in enumerate(py): + lines.extend(ah2.plot(y, p, color=npc.lines[i])) +ah2.set_ybound(lower=0.0) +ah2.legend(lines, ['${0:0.1f}\sigma$'.format(fi) for fi in f]) + +# Logistic ordinal +log_sigma = 0.1 +b = np.linspace(-3.0, 3.0, n) # Log breakpoints +Py_ord = np.zeros((len(f), n), dtype='float') +for i, fi in enumerate(f): + py[i] = beta_likelihood.likelihood(y, fi) + Py[i] = convert_to_discrete(beta_likelihood, fi, n) + + + +plt.show() + + + diff --git a/data/statruns2_oct2017.yaml b/data/statruns2_oct2017.yaml index ba2de29..e87d1cd 100644 --- a/data/statruns2_oct2017.yaml +++ b/data/statruns2_oct2017.yaml @@ -23,15 +23,39 @@ learners: # - name: MaxVar (abs) # model_type: MaxVar # obs_args: {p_rel: 0.0} - - name: DetKSwitch ($p_{rel}=-5.0, \tau=0.1$) (rel, abs) + - name: DetKUCB ($p_{rel}=-2.0, w_v=0.8, \tau=0.1$) (rel, abs) model_type: MaxVar - obs_args: {n_rel_samples: 5, p_rel: -5.0, rel_tau: 0.1, abs_tau: 0.1} - - name: DetKSwitch ($p_{rel}=-5.0, \tau=10^{-5}$) (rel, abs) + obs_args: {n_rel_samples: 5, p_rel: -2.0, rel_tau: 0.1, abs_tau: 0.1, w_v: 0.8} + - name: DetKUCB ($p_{rel}=-2.0, w_v=0.8, \tau=10^{-5}$) (rel, abs) model_type: MaxVar - obs_args: {n_rel_samples: 5, p_rel: -5.0, rel_tau: 1e-5, abs_tau: 1e-5} - - name: DetKUCBHard ($w_v=0.75, \tau=10^{-5}$) (rel, abs) + obs_args: {n_rel_samples: 5, p_rel: -2.0, rel_tau: 1.0e-5, abs_tau: 1.0e-5, w_v: 0.8} + - name: DetKUCB ($p_{rel}=-2.0, w_v=0.8, \tau_r=10^{-5}, \tau_a=0.1$) (rel, abs) model_type: MaxVar - obs_args: {n_rel_samples: 5, p_rel: -5.0, rel_tau: 1e-5, abs_tau: 1e-5, w_v: 0.75} - - name: DetKUCB ($w_v=0.75, \tau=0.1$) (rel, abs) + obs_args: {n_rel_samples: 5, p_rel: -2.0, rel_tau: 0.1, abs_tau: 1.0e-5, w_v: 0.8} + - name: DetKUCB ($p_{rel}=-2.0, w_v=0.8, \tau_r=0.1, \tau_a=10^{-5}$) (rel, abs) model_type: MaxVar - obs_args: {n_rel_samples: 5, p_rel: -5.0, rel_tau: 0.1, abs_tau: 0.1, w_v: 0.75} + obs_args: {n_rel_samples: 5, p_rel: -2.0, rel_tau: 0.1, abs_tau: 1.0e-5, w_v: 0.8} + - name: DetKUCB ($p_{rel}=-5.0, w_v=0.8, \tau=0.1$) (rel, abs) + model_type: MaxVar + obs_args: {n_rel_samples: 5, p_rel: -5.0, rel_tau: 0.1, abs_tau: 0.1, w_v: 0.8} + - name: DetKUCB ($p_{rel}=-5.0, w_v=0.8, \tau=10^{-5}$) (rel, abs) + model_type: MaxVar + obs_args: {n_rel_samples: 5, p_rel: -5.0, rel_tau: 1.0e-5, abs_tau: 1.0e-5, w_v: 0.8} + - name: DetKUCB ($p_{rel}=-5.0, w_v=0.8, \tau_r=10^{-5}, \tau_a=0.1$) (rel, abs) + model_type: MaxVar + obs_args: {n_rel_samples: 5, p_rel: -5.0, rel_tau: 0.1, abs_tau: 1.0e-5, w_v: 0.8} + - name: DetKUCB ($p_{rel}=-5.0, w_v=0.8, \tau_r=0.1, \tau_a=10^{-5}$) (rel, abs) + model_type: MaxVar + obs_args: {n_rel_samples: 5, p_rel: -5.0, rel_tau: 0.1, abs_tau: 1.0e-5, w_v: 0.8} + - name: DetKUCB ($p_{rel}=-10.0, w_v=0.8, \tau=0.1$) (rel, abs) + model_type: MaxVar + obs_args: {n_rel_samples: 5, p_rel: -10.0, rel_tau: 0.1, abs_tau: 0.1, w_v: 0.8} + - name: DetKUCB ($p_{rel}=-10.0, w_v=0.8, \tau=10^{-5}$) (rel, abs) + model_type: MaxVar + obs_args: {n_rel_samples: 5, p_rel: -10.0, rel_tau: 1.0e-5, abs_tau: 1.0e-5, w_v: 0.8} + - name: DetKUCB ($p_{rel}=-10.0, w_v=0.8, \tau_r=10^{-5}, \tau_a=0.1$) (rel, abs) + model_type: MaxVar + obs_args: {n_rel_samples: 5, p_rel: -10.0, rel_tau: 0.1, abs_tau: 1.0e-5, w_v: 0.8} + - name: DetKUCB ($p_{rel}=-10.0, w_v=0.8, \tau_r=0.1, \tau_a=10^{-5}$) (rel, abs) + model_type: MaxVar + obs_args: {n_rel_samples: 5, p_rel: -10.0, rel_tau: 0.1, abs_tau: 1.0e-5, w_v: 0.8} diff --git a/ordinal_likelihoods.py b/ordinal_likelihoods.py new file mode 100644 index 0000000..f2b4623 --- /dev/null +++ b/ordinal_likelihoods.py @@ -0,0 +1,76 @@ +import numpy as np +import matplotlib.pyplot as plt +import GPpref +import nice_plot_colors as npc +from scipy.stats import norm +plt.rc('font',**{'family':'serif','sans-serif':['Computer Modern Roman']}) +plt.rc('text', usetex=True) + +def beta_discrete(b, f, n_cat=5): + P = np.zeros(n_cat+1, dtype='float') + for i in range(1, n_cat+1): + P[i] = b.cdf(float(i)/n_cat, f) - P[0:i].sum() + return P[1:] + + +# Examine how continuous variable is converted to discrete categories through beta distribution and logistic ordinal +n_ords = 5 + +n_f = 101 +f = np.linspace(-9.0, 9.0, n_f) + +# Integrated beta +sigma = 1.0 +v = 15.0 +beta_likelihood = GPpref.AbsBoundProbit(sigma=sigma, v=v) +Py = np.zeros((n_f, n_ords), dtype='float') +for i, fi in enumerate(f): + Py[i] = beta_discrete(beta_likelihood, fi, n_ords) + +fh, ah = plt.subplots() +beta_lines = [] +for i, p in enumerate(Py.T): + beta_lines.extend(ah.plot(f, p, color=npc.lines[i])) +ah.legend(beta_lines, ['$p(y={0}|f)$'.format(ni) for ni in range(n_ords)]) + +# Logistic ordinal +log_sigma = 0.5 +b = np.linspace(-1.1, 1.1, n_ords-1) # Log breakpoints +Py_ord = np.zeros((n_f, n_ords), dtype='float') +z = lambda ff, bb: (bb-ff)/log_sigma +Py_ord[:, 0] = norm.cdf(z(f, b[0])) +Py_ord[:, -1] = 1.0-norm.cdf(z(f, b[-1])) +for i in range(1, n_ords-1): + Py_ord[:,i] = norm.cdf(z(f, b[i])) - norm.cdf(z(f, b[i-1])) + +ord_lines = [] +for i, p in enumerate(Py_ord.T): + ord_lines.extend(ah.plot(f, p, color=npc.lines[i], ls='--')) + +ah.set_xlabel('$f(x)$') +ah.set_ylabel('$p(y|f)$') + + +# Testing the Ordinal Probit class +orpro = GPpref.OrdinalProbit(log_sigma, b[-1], n_ords, eps=1.0e-5) +Py_c = np.zeros((n_f, n_ords), dtype='float') +dPy_c = np.zeros((n_f, n_ords), dtype='float') +d2Py_c = np.zeros((n_f, n_ords), dtype='float') +fv = np.atleast_2d(f).T +y = np.ones(fv.shape, dtype='int') +for i in range(n_ords): + d2p, dp, p = orpro.derivatives(y*(i+1), fv) + d2Py_c[:,i], dPy_c[:,i], Py_c[:,i] = d2p.diagonal(), dp.flat, p.flat + +fh2, ah2 = plt.subplots(1, 3) +ah2[0].plot(f, np.exp(Py_c)) +ah2[0].set_title('$P(y|f(x))$') +ah2[1].plot(f, dPy_c) +ah2[1].set_title('$\partial l(y,f(x)) / \partial f(x)$') +ah2[2].plot(f, d2Py_c) +ah2[2].set_title('$\partial^2 l(y,f(x)) / \partial f(x)^2$') + +plt.show() + + + diff --git a/plot_statruns.py b/plot_statruns.py index 3e64dac..70f6be5 100644 --- a/plot_statruns.py +++ b/plot_statruns.py @@ -54,9 +54,9 @@ def plot_results(wrms_results, true_pos_results, selected_error, obs_array, rela selected_error=selected_error[methods_indexes,:,:] - f0, ax0 = single_plot(wrms_results, names=names, ylabel='Weighted RMSE', bars=bars) - f1, ax1 = single_plot(true_pos_results, names=names, ylabel='True positive selections (out of 15)', bars=True, precut=1, percentile=0) - f2, ax2 = single_plot(selected_error, names=names, ylabel='RMSE of best paths', bars=True, precut=1) + f0, ax0 = single_plot(wrms_results, names=names, ylabel='Weighted RMSE', bars=bars, percentile=-1) + f1, ax1 = single_plot(true_pos_results, names=names, ylabel='True positive selections (out of 30)', bars=True, precut=1, percentile=-1) + f2, ax2 = single_plot(selected_error, names=names, ylabel='RMSE of best paths', bars=True, precut=1, percentile=-1) f = [f0, f1, f2] ax = [ax0, ax1, ax2] @@ -73,12 +73,12 @@ def plot_results(wrms_results, true_pos_results, selected_error, obs_array, rela fr, axr = single_plot(relative_error, names=names, ylabel='Mean relative prediction error', bars=True) f.append(fr); ax.append(axr) if data_dir is not None: - fr.savefig(data_dir + '/rel_error.pdf', bbox_inches='tight') + fr.savefig(data_dir + 'rel_error.pdf', bbox_inches='tight') if data_dir is not None: - f0.savefig(data_dir + '/wrms.pdf', bbox_inches='tight') - f1.savefig(data_dir + '/true_pos.pdf', bbox_inches='tight') - f2.savefig(data_dir + '/rms_best.pdf', bbox_inches='tight') + f0.savefig(data_dir + 'wrms.pdf', bbox_inches='tight') + f1.savefig(data_dir + 'true_pos.pdf', bbox_inches='tight') + f2.savefig(data_dir + 'rms_best.pdf', bbox_inches='tight') # hl = [] # for i in range(mean_err.shape[0]): # hl.append(plt.errorbar(np.arange(mean_err.shape[1]), mean_err[i,:], yerr=std_err[i, :])) @@ -86,40 +86,55 @@ def plot_results(wrms_results, true_pos_results, selected_error, obs_array, rela plt.show() return f + +def _save_file(file, data): + with open(file, 'wb') as fh: + pickle.dump(data, fh) + return + + +def save_data(data_dir, wrms, true_pos, sel_err, obs, rel_err=None): + _save_file(data_dir + 'wrms.pkl', wrms) + _save_file(data_dir + 'true_pos.pkl', true_pos) + _save_file(data_dir + 'selected_error.pkl', sel_err) + _save_file(data_dir + 'obs.pkl', obs) + if rel_err is not None: + _save_file(data_dir + 'relative_error.pkl', rel_err) + + +def _load_file(file): + with open(file, 'rb') as fh: + data = pickle.load(fh) # Dimensions n_learners, n_queries+1, n_trials + return data + + def load_data(data_dir=None): if data_dir is None: Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing data_dir = askdirectory(initialdir='./data/') if data_dir == '': # Cancel clicked raise IOError('No directory selected') + elif data_dir[-1] is not '/': + data_dir += '/' - with open(data_dir+'/wrms.pkl', 'rb') as fh: - wrms_results = pickle.load(fh) # Dimensions n_learners, n_queries+1, n_trials - - with open(data_dir+'/true_pos.pkl', 'rb') as fh: - true_pos_results = pickle.load(fh) - - with open(data_dir+'/selected_error.pkl', 'rb') as fh: - selected_error = pickle.load(fh) - - with open(data_dir+'/obs.pkl', 'rb') as fh: - obs_array = pickle.load(fh) - + wrms_results = _load_file(data_dir+'wrms.pkl') + true_pos_results = _load_file(data_dir+'true_pos.pkl') + selected_error = _load_file(data_dir+'selected_error.pkl') + obs_array = _load_file(data_dir+'obs.pkl') try: - with open(data_dir + '/relative_error.pkl', 'rb') as fh: - relative_error = pickle.load(fh) + relative_error = _load_file(data_dir+'relative_error.pkl') except IOError: print "No relative error data found." relative_error = None - return wrms_results, true_pos_results, selected_error, obs_array, relative_error + return data_dir, wrms_results, true_pos_results, selected_error, obs_array, relative_error def load_and_plot(save_plots=True, *args, **kwargs): - Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing - data_dir = askdirectory(initialdir='./data/') - if data_dir == '': return None # Cancel clicked - wrms_results, true_pos_results, selected_error, obs_array, relative_error = load_data(data_dir) + try: + data_dir, wrms_results, true_pos_results, selected_error, obs_array, relative_error = load_data() + except IOError: + return None if save_plots is False: data_dir = None @@ -128,7 +143,7 @@ def load_and_plot(save_plots=True, *args, **kwargs): return hf def get_selection(): - wrms, true_pos, sel_err, obs, rel_err = load_data() + _dd, wrms, true_pos, sel_err, obs, rel_err = load_data() print "The following models were found:" for i, obs_item in enumerate(obs): print "{0}: {1}".format(i, obs_item['name']) @@ -137,20 +152,26 @@ def get_selection(): if rel_err is not None: rel_err = rel_err[dex, :, :] obs = [obs[i] for i in dex] - finished = input('Finished? (0/1)') - return wrms, true_pos, sel_err, obs, rel_err, finished + # finished = input('Finished? (0/1)') + return wrms, true_pos, sel_err, obs, rel_err def load_multiple(*args, **kwargs): - wrms, true_pos, sel_err, obs, rel_err, finished = get_selection() + wrms, true_pos, sel_err, obs, rel_err = get_selection() app = lambda a, b: np.append(a, b, axis=0) - while not finished: - wrms2, true_pos2, sel_err2, obs2, rel_err2, finished = get_selection() + while True: + try: + wrms2, true_pos2, sel_err2, obs2, rel_err2 = get_selection() + except IOError: + break wrms, true_pos, sel_err = app(wrms, wrms2), app(true_pos, true_pos2), app(sel_err, sel_err2) if rel_err is not None: rel_err = app(rel_err, rel_err2) obs.extend(obs2) hf = plot_results(wrms, true_pos, sel_err, obs, relative_error=rel_err, *args, **kwargs) plt.show() + if 'data_dir' in kwargs: + save_data(kwargs['data_dir'], wrms, true_pos, sel_err, obs, rel_err) + print 'Data saved to {0}'.format(kwargs['data_dir']) return hf, wrms, true_pos, sel_err, obs, rel_err if __name__ == "__main__": diff --git a/pref_active_learning.py b/pref_active_learning.py index 62c6383..3dc52c3 100644 --- a/pref_active_learning.py +++ b/pref_active_learning.py @@ -15,7 +15,7 @@ save_plots = False -with open('./data/mid_freq_5.yaml', 'rt') as fh: +with open('./data/statruns2_oct2017.yaml', 'rt') as fh: wave = yaml.safe_load(fh) try: @@ -25,7 +25,7 @@ n_rel_train = 0 n_abs_train = 1 -n_queries = 50 +n_queries = 20 delta_f = 1e-5 @@ -82,32 +82,17 @@ # Construct active learner object GP_kwargs = {'x_rel':x_rel, 'uvi_rel':uvi_rel, 'x_abs':x_abs, 'y_rel':y_rel, 'y_abs':y_abs, 'delta_f':delta_f, 'rel_likelihood':GPpref.PrefProbit(), 'abs_likelihood':GPpref.AbsBoundProbit()} - -# learner = active_learners.UCBAbsRelD(**active_args) -# # obs_arguments = {'req_improvement': 0.60, 'n_test': 50, 'gamma': 2.0, 'n_rel_samples': 5, 'p_thresh': 0.7} -# obs_arguments = {'n_test': 100, 'p_rel': 0.5, 'n_rel_samples': 5, 'gamma': 2.0} - -# learner = active_learners.ExpectedImprovementRel(**GP_kwargs) -# obs_arguments = {'n_test': 100, 'zeta': 0.1, 'p_rel':1.0} - -# learner = active_learners.SampledClassification(verbose=verbose, **GP_kwargs) -# obs_arguments = {'n_test':50, 'n_samples':20, 'y_threshold':0.7, 'p_pref_tol':1e-3, 'n_mc_abs':50} - -# learner = active_learners.DetSelect(**GP_kwargs) -# obs_arguments = {'n_test': 100, 'n_rel_samples': 5, 'gamma': 2.0, 'tau': 0.5} - -# learner = active_learners.ActiveLearner(**GP_kwargs) -# obs_arguments = {'p_rel':0.0, 'n_rel_samples': 5} - -learner = active_learners.MaxVar(**GP_kwargs) -obs_arguments = {'n_rel_samples': 5, 'p_rel': -1, 'rel_tau': 0.01, 'abs_tau': 0.01} +learner_kwargs = wave['learners'][0] +learner = active_learners.Learner(**learner_kwargs) +learner.build_model(GP_kwargs) +learner.model.set_hyperparameters(log_hyp) # Get initial solution -learner.set_hyperparameters(log_hyp) -f = learner.solve_laplace() -learner.print_hyperparameters() +learner.model.set_hyperparameters(log_hyp) +f = learner.model.solve_laplace() +learner.model.print_hyperparameters() -fig_p, (ax_p_l, ax_p_a, ax_p_r) = learner.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, +fig_p, (ax_p_l, ax_p_a, ax_p_r) = learner.model.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, abs_y_samples, mc_samples) if save_plots: pdf_pages.savefig(fig_p, bbox_inches='tight') @@ -116,34 +101,54 @@ for obs_num in range(n_queries): # if 'p_rel' in obs_arguments: # obs_arguments['p_rel'] = max(0.0, (n_queries - obs_num) / float(n_queries)) - next_x = learner.select_observation(**obs_arguments) + next_x = learner.select_observation() if next_x.shape[0] == 1: next_y, next_f = abs_obs_fun.generate_observations(next_x) - learner.add_observations(next_x, next_y, keep_f=keep_f) + learner.model.add_observations(next_x, next_y, keep_f=keep_f) print 'Abs: x:{0}, y:{1}'.format(next_x[0], next_y[0]) else: next_y, next_uvi, next_fx = rel_obs_fun.gaussian_multi_pairwise_sampler(next_x) next_fuv = next_fx[next_uvi][:, :, 0] fuv_rel = np.concatenate((fuv_rel, next_fuv), 0) - learner.add_observations(next_x, next_y, next_uvi, keep_f=keep_f) + learner.model.add_observations(next_x, next_y, next_uvi, keep_f=keep_f) print 'Rel: x:{0}, best_index:{1}'.format(next_x.flatten(), next_uvi[0, 1]) - f = learner.solve_laplace() + f = learner.model.solve_laplace() if save_plots: - fig_p, (ax_p_l, ax_p_a, ax_p_r) = learner.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, + fig_p, (ax_p_l, ax_p_a, ax_p_r) = learner.model.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, abs_y_samples, mc_samples) pdf_pages.savefig(fig_p, bbox_inches='tight') # fig_p.savefig(fig_dir+'posterior{0:02d}.pdf'.format(obs_num+1), bbox_inches='tight') plt.close(fig_p) -learner.print_hyperparameters() +learner.model.print_hyperparameters() if save_plots: pdf_pages.close() else: - fig_p, (ax_p_l, ax_p_a, ax_p_r) = learner.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, + fig_post, (ax_p_l, ax_p_a, ax_p_r) = learner.model.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, abs_y_samples, mc_samples) plt.show() -print "Finished!" \ No newline at end of file +print "Finished!" + +## SCRAP +# learner = active_learners.UCBAbsRelD(**active_args) +# # obs_arguments = {'req_improvement': 0.60, 'n_test': 50, 'gamma': 2.0, 'n_rel_samples': 5, 'p_thresh': 0.7} +# obs_arguments = {'n_test': 100, 'p_rel': 0.5, 'n_rel_samples': 5, 'gamma': 2.0} + +# learner = active_learners.ExpectedImprovementRel(**GP_kwargs) +# obs_arguments = {'n_test': 100, 'zeta': 0.1, 'p_rel':1.0} + +# learner = active_learners.SampledClassification(verbose=verbose, **GP_kwargs) +# obs_arguments = {'n_test':50, 'n_samples':20, 'y_threshold':0.7, 'p_pref_tol':1e-3, 'n_mc_abs':50} + +# learner = active_learners.DetSelect(**GP_kwargs) +# obs_arguments = {'n_test': 100, 'n_rel_samples': 5, 'gamma': 2.0, 'tau': 0.5} + +# learner = active_learners.ActiveLearner(**GP_kwargs) +# obs_arguments = {'p_rel':0.0, 'n_rel_samples': 5} + +# learner = active_learners.MaxVar(**GP_kwargs) +# obs_arguments = {'n_rel_samples': 5, 'p_rel': -10.0, 'rel_tau': 0.1, 'abs_tau': 0.1} From 231ecfa62b8d40c00576835f7aa3a7efe5cdf063 Mon Sep 17 00:00:00 2001 From: Nicholas Lawrance Date: Fri, 17 Nov 2017 17:39:22 -0800 Subject: [PATCH 19/38] Ordinals likelihood working, changed the y_list in OrdinalProbit to a column --- GP_preference_demo.py | 31 ++----- GPpref.py | 176 ++++++++++++++++++++++++++---------- active_learners.py | 149 ++++++++++++++++++++++++++---- active_statruns.py | 14 ++- data/statruns2_oct2017.yaml | 37 +------- ordinal_likelihoods.py | 78 +++++++++++++--- plot_tools.py | 27 +++--- pref_active_learning.py | 66 +++++++------- 8 files changed, 391 insertions(+), 187 deletions(-) diff --git a/GP_preference_demo.py b/GP_preference_demo.py index 47b8731..0054b22 100644 --- a/GP_preference_demo.py +++ b/GP_preference_demo.py @@ -14,7 +14,7 @@ use_test_data = False # test_data.data3 # verbose = 2 -with open('./data/statruns_nov2017.yaml', 'rt') as fh: +with open('./data/ordinal_test.yaml', 'rt') as fh: wave = yaml.safe_load(fh) try: @@ -24,29 +24,24 @@ random_wave = test_data.MultiWave(**wave['wave_params']) log_hyp = np.log(wave['hyperparameters']) -n_rel_train = 5 -n_abs_train = 15 - -delta_f = 1e-5 +n_rel_train = 10 +n_abs_train = 20 n_xplot = 101 -n_mcsamples = 1000 -n_ysamples = 301 n_posterior_samples = 3 random_wave.print_values() true_function = random_wave.out -rel_obs_fun = GPpref.RelObservationSampler(true_function, GPpref.PrefProbit(**wave['rel_obs_params'])) -abs_obs_fun = GPpref.AbsObservationSampler(true_function, GPpref.AbsBoundProbit(**wave['abs_obs_params'])) +rel_obs_fun = GPpref.RelObservationSampler(true_function, wave['GP_params']['rel_likelihood'], wave['rel_obs_params']) +abs_obs_fun = GPpref.AbsObservationSampler(true_function, wave['GP_params']['abs_likelihood'], wave['abs_obs_params']) # True function x_plot = np.linspace(0.0,1.0,n_xplot,dtype='float') x_test = np.atleast_2d(x_plot).T f_true = abs_obs_fun.f(x_test) mu_true = abs_obs_fun.mean_link(x_test) -mc_samples = np.random.normal(size=n_mcsamples) -abs_y_samples = np.atleast_2d(np.linspace(0.01, 0.99, n_ysamples)).T +abs_y_samples = abs_obs_fun.l.y_list p_abs_y_true = abs_obs_fun.observation_likelihood_array(x_test, abs_y_samples) p_rel_y_true = rel_obs_fun.observation_likelihood_array(x_test) @@ -60,10 +55,8 @@ x_abs, y_abs, mu_abs = abs_obs_fun.generate_n_observations(n_abs_train) # Construct GP object -prefGP = GPpref.PreferenceGaussianProcess(x_rel, uvi_rel, x_abs, y_rel, y_abs, - delta_f=delta_f, - rel_likelihood=GPpref.PrefProbit(), - abs_likelihood=GPpref.AbsBoundProbit(), verbose=verbose) +wave['GP_params']['verbose'] = verbose +prefGP = GPpref.PreferenceGaussianProcess(x_rel, uvi_rel, x_abs, y_rel, y_abs, **wave['GP_params']) prefGP.set_hyperparameters(log_hyp) # If training hyperparameters, use external optimiser @@ -76,18 +69,14 @@ # Latent predictions fhat, vhat = prefGP.predict_latent(x_test) -# Expected values -E_y = prefGP.abs_posterior_mean(x_test, fhat, vhat) - -# Posterior likelihoods (MC sampled for absolute) -p_abs_y_post = prefGP.abs_posterior_likelihood(abs_y_samples, fhat=fhat, varhat=vhat, normal_samples=mc_samples) +# Posterior likelihoods +p_abs_y_post, E_y = prefGP.abs_posterior_likelihood(abs_y_samples, fhat=fhat, varhat=vhat) p_rel_y_post = prefGP.rel_posterior_likelihood_array(fhat=fhat, varhat=vhat) # Plot true functions fig_t, (ax_t_l, ax_t_a, ax_t_r) = ptt.true_plots(x_test, f_true, mu_true, wave['rel_obs_params']['sigma'], abs_y_samples, p_abs_y_true, p_rel_y_true, - x_abs, y_abs, uv_rel, fuv_rel, y_rel, t_l=r'True latent function, $f(x)$') # Posterior estimates diff --git a/GPpref.py b/GPpref.py index 551fc51..1dbfce8 100644 --- a/GPpref.py +++ b/GPpref.py @@ -5,6 +5,7 @@ from scipy.special import digamma, polygamma from scipy.stats import norm, beta from scipy.linalg import block_diag +import sys class LaplaceException(Exception): pass @@ -73,10 +74,17 @@ def compute_Kxz_matrix(self, z): return Kxz class PrefProbit(object): + type = 'preference' + y_type = 'discrete' + y_list = np.array([-1, 1], dtype='int') + def __init__(self, sigma=1.0): - self.set_sigma(sigma) + self.set_hyper([sigma]) self.log2pi = np.log(2.0*np.pi) + def set_hyper(self, hyper): + self.set_sigma(hyper[0]) + def set_sigma(self, sigma): self.sigma = sigma self._isqrt2sig = 1.0 / (self.sigma * np.sqrt(2.0)) @@ -152,13 +160,20 @@ def generate_samples(self, f): class OrdinalProbit(object): + type = 'categorical' + y_type = 'discrete' + def __init__(self, sigma=1.0, b=1.0, n_ordinals=5, eps=1.0e-10): - self.set_sigma(sigma) - self.set_b(b, n_ordinals) + self.n_ordinals=n_ordinals + self.set_hyper([sigma, b]) self.eps = eps + self.y_list = np.atleast_2d(np.arange(1, self.n_ordinals+1, 1, dtype='int')).T + + def set_hyper(self, hyper): + self.set_sigma(hyper[0]) + self.set_b(hyper[1]) - def set_b(self, b, n_ordinals): - self.n_ordinals = n_ordinals + def set_b(self, b): if not hasattr(b, "__len__"): b = abs(b) self.b = np.hstack(([-np.Inf],np.linspace(-b, b, self.n_ordinals-1), [np.Inf])) @@ -183,6 +198,7 @@ def z_k(self, y, f): return self._isigma*(self.b[y] - f) def norm_pdf(self, y, f): + f = f*np.ones(y.shape, dtype='float') # This ensures the number of f values matches the number of y out = np.zeros(y.shape, dtype='float') for i in range(out.shape[0]): if y[i] != 0 and y[i] != self.n_ordinals: # b0 = -Inf -> N(-Inf) = 0 @@ -190,17 +206,20 @@ def norm_pdf(self, y, f): out[i] = std_norm_pdf(z) return out - def norm_cdf(self, y, f): + def norm_cdf(self, y, f, var_x=0.0): + ivar = self._isigma + var_x + f = f*np.ones(y.shape, dtype='float') out = np.zeros(y.shape, dtype='float') for i in range(out.shape[0]): - if y[i] == self.n_ordinals: + if y[i][0] == self.n_ordinals: out[i] = 1.0 - elif y[i] != 0: - z = self._isigma*(self.b[y[i]] - f[i]) + elif y[i][0] != 0: + z = ivar*(self.b[y[i]] - f[i]) out[i] = std_norm_cdf(z) return out def z_pdf(self, y, f): + f = f*np.ones(y.shape, dtype='float') out = np.zeros(y.shape, dtype='float') for i in range(out.shape[0]): if y[i] != 0 and y[i] != self.n_ordinals: # b0 = -Inf -> N(-Inf) = 0 @@ -232,35 +251,75 @@ def derivatives(self, y, f): # d2py_df2[i] = (l2 - 2*l[i] + l0)/self.delta_f**2/dpy_df[i]/l[i] if y[i] == 1: - dpy_df[i] = -self._isigma*self.z_k(y[i], f[i]) - d2py_df2[i] = self._ivar + dpy_df[i] = self._isigma*self.z_k(y[i], f[i]) + d2py_df2[i] = -self._ivar elif y[i] == self.n_ordinals: - dpy_df[i] = -self._isigma*self.z_k(y[i]-1, f[i]) - d2py_df2[i] = self._ivar + dpy_df[i] = self._isigma*self.z_k(y[i]-1, f[i]) + d2py_df2[i] = -self._ivar else: z1 = self.z_k(y[i], f[i]) z2 = self.z_k(y[i]-1, f[i]) ep = np.exp(-0.5*(z1**2 - z2**2)) - dpy_df[i] = -self._isigma*(z1*ep-z2)/(ep - 1.0) - d2py_df2[i] = self._ivar*(1.0 - (z1**2 *ep - z2**2)/(ep - 1.0)) + dpy_df[i]**2 + dpy_df[i] = self._isigma*(z1*ep-z2)/(ep - 1.0) + d2py_df2[i] = -(self._ivar*(1.0 - (z1**2 *ep - z2**2)/(ep - 1.0)) + dpy_df[i]**2) else: - dpy_df[i] = self._isigma*(self.norm_pdf(y[i], f[i]) - self.norm_pdf(y[i]-1, f[i])) / l[i] - d2py_df2[i] = dpy_df[i]**2 + self._ivar*(self.z_pdf(y[i], f[i]) - self.z_pdf(y[i]-1, f[i])) / l[i] + dpy_df[i] = -self._isigma*(self.norm_pdf(y[i], f[i]) - self.norm_pdf(y[i]-1, f[i])) / l[i] + d2py_df2[i] = -(dpy_df[i]**2 + self._ivar*(self.z_pdf(y[i], f[i]) - self.z_pdf(y[i]-1, f[i])) / l[i]) - W = np.diagflat(d2py_df2) + W = np.diagflat(-d2py_df2) return W, dpy_df, py + def posterior_likelihood(self, fhat, var_star, y=None): + if y is None: + y = self.y_list + py = np.zeros((len(y), len(fhat)), dtype='float') + mu = np.zeros(fhat.shape, dtype='float') + for i, (f, v) in enumerate(zip(fhat, var_star.diagonal())): + py[:,[i]] = self.norm_cdf(y, f, v) - self.norm_cdf(y-1, f, v) + mu[i] = (py[:,[i]]*y).sum() + if len(y) != self.n_ordinals or any(y != self.y_list): + print "Specified y vector is not full ordinal set." + mu = self.posterior_mean(fhat, var_star) + return py, mu + + def posterior_mean(self, fhat, var_star): + py, mu = self.posterior_likelihood(fhat, var_star) + return mu + + def mean_link(self, f): + mu = np.zeros(f.shape, dtype='float') + for i in range(f.shape[0]): + py = self.likelihood(self.y_list, f[i]) + mu[i] = (py*self.y_list).sum() # Expected value, first moment + return mu + + def generate_samples(self, f): + z = np.zeros(f.shape, dtype='int') + mu = np.zeros(f.shape, dtype='float') + for i in range(f.shape[0]): + py = self.likelihood(self.y_list, f[i]) + mu[i] = (py*self.y_list).sum() # Expected value, first moment + z[i] = np.sum(np.random.uniform() > py.cumsum())+1 + return z, mu + class AbsBoundProbit(object): + type = 'bounded continuous' + y_type = 'bounded' + y_list = np.linspace(0.01, 0.99, 101) + def __init__(self, sigma=1.0, v=10.0): # v is the precision, kind of related to inverse of noise, high v is sharp distributions # sigma is the slope of the probit, basically scales how far away from # 0 the latent has to be to to move away from 0.5 output. Sigma should # basically relate to the range of the latent function - self.set_sigma(sigma) - self.set_v(v) + self.set_hyper([sigma, v]) self.log2pi = np.log(2.0*np.pi) + def set_hyper(self, hyper): + self.set_sigma(hyper[0]) + self.set_v(hyper[1]) + def set_sigma(self, sigma): self.sigma = sigma self._isqrt2sig = 1.0 / (self.sigma * np.sqrt(2.0)) @@ -317,9 +376,25 @@ def derivatives(self, y, f): return -W, dpy_df, py + def posterior_likelihood(self, fhat, varhat, y=None, normal_samples=None): # This is MC sampled due to no closed_form solution + if normal_samples is None: + normal_samples = np.random.normal(size=1000) + iny = 1.0/len(normal_samples) + + if y is None: + y = self.y_list + + # Sampling from posterior to show likelihoods + p_y = np.zeros((len(y), fhat.shape[0])) + for i, (fstar, vstar) in enumerate(zip(fhat, varhat.diagonal())): + f_samples = normal_samples * vstar + fstar + p_y[:, i] = [iny * np.sum(self.likelihood(yj, f_samples)) for yj in y] + return p_y, self.posterior_mean(fhat, varhat) + def posterior_mean(self, fhat, var_star): #mu_t = self.mean_link(fhat) - E_x = np.clip(std_norm_cdf(fhat/(np.sqrt(2*self.sigma**2 + var_star))), 1e-12, 1.0-1e-12) + vv = np.atleast_2d(var_star.diagonal()).T + E_x = np.clip(std_norm_cdf(fhat/(np.sqrt(2*self.sigma**2 + vv))), 1e-12, 1.0-1e-12) return E_x def generate_samples(self, f): @@ -333,23 +408,30 @@ def generate_samples(self, f): class PreferenceGaussianProcess(object): - def __init__(self, x_rel, uvi_rel, x_abs, y_rel, y_abs, rel_likelihood=PrefProbit(), delta_f=1e-6, - abs_likelihood=AbsBoundProbit(), verbose=False): + def __init__(self, x_rel, uvi_rel, x_abs, y_rel, y_abs, rel_likelihood='PrefProbit', delta_f=1.0e-5, + abs_likelihood='AbsBoundProbit', verbose=False, hyper_counts=[2, 1, 2]): + # hyperparameters are split by hyper_counts, where hyper_counts should contain three integers > 0, the first is + # the number of hypers in the GP covariance, second is the number in the relative likelihood, last is the number + # in the absolute likelihood. hyper_counts.sum() should be equal to len(log_hyp) # log_hyp are log of hyperparameters, note that it is [length_0, ..., length_d, sigma_f, sigma_probit, v_beta] + # Training points are split into relative and absolute for calculating f, but combined for predictions. self.set_observations(x_rel, uvi_rel, x_abs, y_rel, y_abs) self.delta_f = delta_f - self.rel_likelihood = rel_likelihood - self.abs_likelihood = abs_likelihood + self.rel_likelihood = getattr(sys.modules[__name__], rel_likelihood)() # This calls the constructor from string + self.abs_likelihood = getattr(sys.modules[__name__], abs_likelihood)() self.verbose = verbose - self.kern = GPy.kern.RBF(self._xdim, ARD=True) + if self._xdim is not None: + self.kern = GPy.kern.RBF(self._xdim, ARD=True) self.Ix = np.eye(self._nx) self.f = None + + self.hyper_counts = np.array(hyper_counts) self.init_extras() def init_extras(self): @@ -361,7 +443,8 @@ def set_observations(self, x_rel, uvi_rel, x_abs, y_rel, y_abs): elif x_abs.shape[0] is not 0: self._xdim = x_abs.shape[1] else: - raise Exception("No Input Points") + self._xdim = None + # raise Exception("No Input Points") self._n_rel = x_rel.shape[0] self._n_abs = x_abs.shape[0] self.x_rel = x_rel @@ -380,6 +463,9 @@ def add_observations(self, x, y, uvi=None, keep_f=False): # keep_f is used to reset the Laplace solution. If it's a small update, it's sometimes better to keep the old # values and append some 0's for the new observations (keep_f = True), otherwise it is reset (keep_f = False) # Default is keep_f = False + if self._xdim is None: + self._xdim = x.shape[1] + self.kern = GPy.kern.RBF(self._xdim, ARD=True) if uvi is None: # Absolute observation/s if keep_f: self.f = np.vstack((self.f, np.zeros((x.shape[0], 1)))) @@ -395,11 +481,16 @@ def add_observations(self, x, y, uvi=None, keep_f=False): self.set_observations(x_rel, uvi_rel, self.x_abs, y_rel, self.y_abs) def set_hyperparameters(self, loghyp): + assert sum(self.hyper_counts) == len(loghyp), "Sum of hyper_counts must match length of log_hyp" + assert self.hyper_counts[0] == self._xdim+1, "Currently only supporting sq exp covariance, hyper_counts[0] must be x_dim + 1" self.kern.lengthscale = np.exp(loghyp[0:self._xdim]) self.kern.variance = 1.0 # (np.exp(loghyp[self._xdim]))**2 - self.rel_likelihood.set_sigma(np.exp(loghyp[-3])) # Do we need different sigmas for each likelihood? Yes! - self.abs_likelihood.set_sigma(np.exp(loghyp[-2])) # I think this sigma relates to sigma_f in the covariance, and is actually possibly redundant - self.abs_likelihood.set_v(np.exp(loghyp[-1])) # Should this relate to the rel_likelihood probit noise? + dex = self.hyper_counts.cumsum() + self.rel_likelihood.set_hyper(np.exp(loghyp[dex[0]:dex[1]])) + self.abs_likelihood.set_hyper(np.exp(loghyp[dex[1]:dex[2]])) + # self.rel_likelihood.set_sigma(np.exp(loghyp[-3])) # Do we need different sigmas for each likelihood? Yes! + # self.abs_likelihood.set_sigma(np.exp(loghyp[-2])) # I think this sigma relates to sigma_f in the covariance, and is actually possibly redundant + # self.abs_likelihood.set_v(np.exp(loghyp[-1])) # Should this relate to the rel_likelihood probit noise? def calc_laplace(self, loghyp=None): if loghyp is not None: @@ -549,25 +640,13 @@ def rel_posterior_MAP(self, uvi, x=None, fhat=None, varhat=None): p_y = self.rel_posterior_likelihood(uvi, y=1, x=x, fhat=fhat, varhat=varhat) return 2*np.array(p_y < 0.5, dtype='int')-1 - def abs_posterior_likelihood(self, y, x=None, fhat=None, varhat=None, normal_samples = None): # Currently sample-based + def abs_posterior_likelihood(self, y=None, x=None, fhat=None, varhat=None, **kwargs): # Currently sample-based fhat, varhat = self._check_latent_input(x, fhat, varhat) - varhat = np.atleast_2d(varhat.diagonal()).T - if normal_samples is None: - normal_samples = np.random.normal(size=1000) - iny = 1.0/len(normal_samples) - - # Sampling from posterior to show likelihoods - p_y = np.zeros((y.shape[0], fhat.shape[0])) - for i, (fstar, vstar) in enumerate(zip(fhat, varhat)): - f_samples = normal_samples * vstar + fstar - p_y[:, i] = [iny * np.sum(self.abs_likelihood.likelihood(yj, f_samples)) for yj in y] - return p_y + return self.abs_likelihood.posterior_likelihood(fhat, varhat, y=y, **kwargs) def abs_posterior_mean(self, x=None, fhat=None, varhat=None): fhat, varhat = self._check_latent_input(x, fhat, varhat) - varstar = np.atleast_2d(varhat.diagonal()).T - E_y = self.abs_likelihood.posterior_mean(fhat, varstar) - return E_y + return self.abs_likelihood.posterior_mean(fhat, varhat) def print_hyperparameters(self): print "COV: '{0}', l: {1}, sigma_f: {2}".format(self.kern.name, self.kern.lengthscale.values, np.sqrt(self.kern.variance.values)) @@ -578,9 +657,10 @@ def print_hyperparameters(self): class ObservationSampler(object): - def __init__(self, true_fun, likelihood_object): + def __init__(self, true_fun, likelihood_type, likelihood_kwargs): self.f = true_fun - self.l = likelihood_object + ltype = getattr(sys.modules[__name__], likelihood_type) + self.l = ltype(**likelihood_kwargs) def generate_observations(self, x): fx = self.f(x) @@ -623,6 +703,8 @@ def generate_n_observations(self, n, n_xdim=1, domain=None): y, mu = self.generate_observations(x) return x, y, mu + def get_y_samples(self): + return self.l.get_y_samples() class RelObservationSampler(ObservationSampler): def observation_likelihood_array(self, x, y=-1): diff --git a/active_learners.py b/active_learners.py index 7aeb817..c0a37aa 100644 --- a/active_learners.py +++ b/active_learners.py @@ -23,8 +23,8 @@ def __init__(self, model_type, obs_args, name, update_p_rel = False): self.name = name self.update_p_rel = update_p_rel - def build_model(self, training_data): - self.model = self.model_type(**training_data) + def build_model(self, model_kwargs): + self.model = self.model_type(**model_kwargs) def select_observation(self): return self.model.select_observation(**self.obs_arguments) @@ -64,16 +64,11 @@ def linear_domain_sampler(self, n_samples, domain=None): x_test = x_test*np.diff(domain, axis=0) + domain[0, :] return x_test - def create_posterior_plot(self, x_test, f_true, mu_true, rel_sigma, fuv_train, abs_y_samples, mc_samples): + def create_posterior_plot(self, x_test, f_true, mu_true, rel_sigma, fuv_train, abs_y_samples): # Latent predictions fhat, vhat = self.predict_latent(x_test) - # Expected values - E_y = self.abs_posterior_mean(x_test, fhat, vhat) - - # Absolute posterior likelihood (MC sampled) - # Posterior likelihoods (MC sampled for absolute) - p_abs_y_post = self.abs_posterior_likelihood(abs_y_samples, fhat=fhat, varhat=vhat, normal_samples=mc_samples) + p_abs_y_post, E_y = self.abs_posterior_likelihood(abs_y_samples, fhat=fhat, varhat=vhat) p_rel_y_post = self.rel_posterior_likelihood_array(fhat=fhat, varhat=vhat) x_train, uvi_train, x_abs_train, y_train, y_abs_train = self.get_observations() uv_train = x_train[uvi_train][:, :, 0] @@ -89,8 +84,17 @@ def create_posterior_plot(self, x_test, f_true, mu_true, rel_sigma, fuv_train, a class MaxVar(ActiveLearner): + l2pie = 1.0 + np.log(2*np.pi) + def logistic(self, H, n_rel): + L = 2.0 + return 1.0 / (1.0 + np.exp(-L * (H-self.kern.variance)) ) + # Max variance - def select_observation(self, domain=None, n_test=100, n_rel_samples = 2, p_rel = 0.0, rel_tau = 1.0, abs_tau = 1.0e-5,w_v=1.0): + def select_observation(self, domain=None, n_test=100, n_rel_samples = 2, p_rel = 0.0, rel_tau = 1.0, abs_tau = 1.0e-5,w_v=1.0, selector='det'): + # p_rel is the likelihood of selecting a relative query + # tau are softmax temperatures (low is more greedy, high is more random) + # w_v is the UCB weighting for variance, where UCB = w_v*variance + (1-w_v)*mean + # det_type is the type of value used to select between # If p_rel < 0 then we will select based on variance magnitude if p_rel < 0.0: p_select = 1.0 @@ -100,28 +104,35 @@ def select_observation(self, domain=None, n_test=100, n_rel_samples = 2, p_rel = fhat, vhat = self.predict_latent(x_test) vv = np.sqrt(np.diagonal(vhat)) - if np.random.uniform() < p_select: # i.e choose a relative sample + if p_select >= 1.0 or np.random.uniform() < p_select: # i.e choose a relative sample available_indexes = set(range(len(x_test))) dK = np.zeros(len(available_indexes)) best_n = [softmax_selector(vv, tau=rel_tau)] while len(best_n) < n_rel_samples: - dK[best_n[-1]] = -1e5 # Bit of a scam because it is still possible to sample this value + dK[best_n[-1]] = -1.0e10 # Bit of a scam because it is still possible to sample this value available_indexes.remove(best_n[-1]) best_n.append(-1) for cn in available_indexes: best_n[-1] = cn K = vhat[np.ix_(best_n, best_n)] dK[cn] = np.linalg.det(K) - best_n[-1] = softmax_selector(dK, tau=rel_tau) # np.argmax(dK) # + best_n[-1] = softmax_selector(dK/dK.max(), tau=rel_tau) # np.argmax(dK) # else: - best_n = [softmax_selector(vv, tau=abs_tau)] #[np.argmax(ucb)] # + best_n = [softmax_selector(w_v*vv+(1.0-w_v)*fhat.flatten(), tau=abs_tau)] #[np.argmax(ucb)] # # This chooses an absolute query based on determinant. Choose relK using beta cdf likelihood - if p_rel <= -1.0: - best_detK = -p_rel*np.sqrt(dK[best_n[-1]]) - K_ratio = best_detK/(vv.max() + best_detK) - if np.random.uniform() > beta.cdf(K_ratio, -p_rel, -p_rel): - best_n = [softmax_selector(w_v*vv+(1.0-w_v)*fhat.flatten(), tau=abs_tau)] + if p_rel < 0.0: + best_abs = softmax_selector(w_v * vv + (1.0 - w_v) * fhat.flatten(), tau=abs_tau) + if selector == 'det': + best_detK = -p_rel*np.sqrt(dK[best_n[-1]]) + K_ratio = best_detK/(vv.max() + best_detK) + elif selector == 'entropy': + H_rel = 0.5*(n_rel_samples*self.l2pie + np.log(dK[best_n[-1]])) + H_abs = 0.5*(self.l2pie + np.log(vhat[best_abs, best_abs])) + K_ratio = H_rel/(H_rel + H_abs) + p_select = beta.cdf(K_ratio, -p_rel, -p_rel) + if np.random.uniform() > p_select: + best_n = [best_abs] return x_test[best_n, :] class UCBLatent(ActiveLearner): @@ -578,4 +589,102 @@ def calculate_threshold_utility(self, fhat, vhat, n_samples, y_threshold): predict_below = fm < y_threshold predicted_accuracy = P_below[predict_below].sum() + (1.0 - P_below[~predict_below]).sum() threshold_utility += predicted_accuracy/len(fhat) - return threshold_utility/n_samples \ No newline at end of file + return threshold_utility/n_samples + + +class OrdinalSampler(PeakComparitor): + def model_value(self, fhat, vhat): + # Probability of missclassification + p_y, mu = self.abs_likelihood.posterior_likelihood(fhat, vhat) + + pass + + + def test_observation(self, x, y, uvi, x_test, n_samples, y_threshold, f = None): + self.add_observations(x, y, uvi, keep_f=True) + self.solve_laplace() + fhat, vhat = self.predict_latent(x_test) + util = self.model_value(fhat, vhat) + self.reset_observations() + return util + + def select_observation(self, domain=None, x_test=None, n_test=50, n_samples=50, y_threshold=0.8, p_pref_tol=1e-3, n_mc_abs=5): + # n_test is the number of test point locations on the input function + # n_samples is the number of functions sampled from the posterior for estimating the utility + # n_mc_abs is the number of proposed observations sampled from the current posterior for absolute estimates + # Generate a set of test points in the domain (if not specified) + if x_test is None: + # x_test = self.linear_domain_sampler(n_test, domain) + x_test = self.uniform_domain_sampler(n_test, domain) + # x_test.sort(axis=0) + n_test = len(x_test) + + # Sample a set of functions from the current posterior + # We save the f value because otherwise it gets out of whack when we add observations + flap = self.solve_laplace() + fhat, vhat = self.predict_latent(x_test) + # base_utility = self.calculate_threshold_utility(fhat, vhat, n_samples, y_threshold) + + # Check a random set of pairwise relatives and all absolutes (massive sampling) + + # Generate a set of random pairs (this randomly pairs all x_test points) + uvi = np.random.choice(n_test, (n_test/2, 2), replace=False) + p_pref = self.rel_likelihood.posterior_likelihood(fhat, vhat, uvi, y=-1) + V_max_rel = 0.0 + + self.store_observations() + + # Relative observations + t_rel = time.time() + # Now calculate the expected value for each observation pair + for i, uv in enumerate(uvi): + x = x_test[uv] + V_rel = 0.0 + try: + if p_pref[i] < p_pref_tol: + V_rel += (1-p_pref[i])*self.test_observation(x, self._plus_y_obs, self._default_uvi, x_test, n_samples, y_threshold, f=flap) + elif p_pref[i] > 1.0-p_pref_tol: + V_rel += p_pref[i]*self.test_observation(x, self._minus_y_obs, self._default_uvi, x_test, n_samples, y_threshold, f=flap) + else: + V_rel += (1-p_pref[i])*self.test_observation(x, self._plus_y_obs, self._default_uvi, x_test, n_samples, y_threshold, f=flap) + V_rel += p_pref[i] * self.test_observation(x, self._minus_y_obs, self._default_uvi, x_test, n_samples, y_threshold, f=flap) + if V_rel >= V_max_rel: + V_max_rel = V_rel + x_best_rel = x + except GPpref.LaplaceException as exc: + print "Failed in relative test observation, x = [{0}, {1}]".format(x[0], x[1]) + raise exc + + # best_n = np.argpartition(V, -n_comparators)[-n_comparators:] + # best = np.argmax(V) + if self.verbose: + print 'V_max_rel = {0}, x = {2}, t = {1}s'.format(V_max_rel[0], time.time()-t_rel, x_best_rel[:,0]) + + # Absolute queries + V_max = 0.0 + t_rel = time.time() + for i, x in enumerate(x_test): + F = fhat[i] + np.random.randn(n_mc_abs)*np.sqrt(vhat[i,i]) + Y, mu = self.abs_likelihood.generate_samples(F) + V_abs = 0.0 + Y = np.clip(Y, 1e-2, 1-1e-2) # I had stability problems in Laplace with values approaching 0 or 1 + for y in Y: + try: + V_abs += self.test_observation(x, y, None, x_test, n_samples, y_threshold, f=flap) + except ValueError: + print "NaN Issue" + except GPpref.LaplaceException as exc: + print "Failed in absolute test observation, x = {0}, y = {1}".format(x, y) + raise exc + V_abs /= n_mc_abs + if V_abs > V_max: + V_max = V_abs + x_best = x + if self.verbose: + print 'V_max_abs = {0}, x = {2}, t = {1}s'.format(V_max, time.time() - t_rel, x_best) + + if V_max_rel > V_max: + x_best = x_best_rel + + return x_best + diff --git a/active_statruns.py b/active_statruns.py index e7b3622..f8a97d8 100644 --- a/active_statruns.py +++ b/active_statruns.py @@ -28,8 +28,7 @@ else: calc_relative_error = False -# Learner parameters -n_rel_samples = run_parameters['learner_params']['n_rel_samples'] +n_rel_samples = run_parameters['statrun_params']['n_rel_samples'] # Define polynomial function to be modelled random_wave = test_data.MultiWave(**run_parameters['wave_params']) @@ -76,8 +75,8 @@ random_wave.randomize() random_wave.print_values() waver.set_vals(trial_number, *random_wave.get_values()) - rel_obs_fun = GPpref.RelObservationSampler(random_wave.out, GPpref.PrefProbit(**run_parameters['rel_obs_params'])) - abs_obs_fun = GPpref.AbsObservationSampler(random_wave.out, GPpref.AbsBoundProbit(**run_parameters['abs_obs_params'])) + rel_obs_fun = GPpref.RelObservationSampler(random_wave.out, run_parameters['GP_params']['rel_likelihood'], run_parameters['rel_obs_params']) + abs_obs_fun = GPpref.AbsObservationSampler(random_wave.out, run_parameters['GP_params']['abs_likelihood'], run_parameters['abs_obs_params']) f_true = abs_obs_fun.f(x_test) y_abs_true = abs_obs_fun.mean_link(x_test) @@ -89,13 +88,12 @@ # Initial data x_rel, uvi_rel, uv_rel, y_rel, fuv_rel = rel_obs_fun.generate_n_observations(statrun_params['n_rel_train'], n_xdim=1) x_abs, y_abs, mu_abs = abs_obs_fun.generate_n_observations(statrun_params['n_abs_train'], n_xdim=1) - training_data = {'x_rel': x_rel, 'uvi_rel': uvi_rel, 'x_abs': x_abs, 'y_rel': y_rel, 'y_abs': y_abs, - 'delta_f': run_parameters['learner_params']['delta_f'], 'rel_likelihood': GPpref.PrefProbit(), - 'abs_likelihood': GPpref.AbsBoundProbit()} + model_kwargs = {'x_rel': x_rel, 'uvi_rel': uvi_rel, 'x_abs': x_abs, 'y_rel': y_rel, 'y_abs': y_abs} + model_kwargs.update(run_parameters['GP_params']) # Get initial solution for nl, learner in enumerate(learners): - learner.build_model(training_data) + learner.build_model(model_kwargs) learner.model.set_hyperparameters(log_hyp) f = learner.model.solve_laplace() fhat, vhat = learner.model.predict_latent(x_test) diff --git a/data/statruns2_oct2017.yaml b/data/statruns2_oct2017.yaml index e87d1cd..6501171 100644 --- a/data/statruns2_oct2017.yaml +++ b/data/statruns2_oct2017.yaml @@ -23,39 +23,6 @@ learners: # - name: MaxVar (abs) # model_type: MaxVar # obs_args: {p_rel: 0.0} - - name: DetKUCB ($p_{rel}=-2.0, w_v=0.8, \tau=0.1$) (rel, abs) + - name: HUCB ($p_{rel}=-1.0, w_v=0.75, \tau_r=0.1, \tau_a=0.05$) (rel, abs) model_type: MaxVar - obs_args: {n_rel_samples: 5, p_rel: -2.0, rel_tau: 0.1, abs_tau: 0.1, w_v: 0.8} - - name: DetKUCB ($p_{rel}=-2.0, w_v=0.8, \tau=10^{-5}$) (rel, abs) - model_type: MaxVar - obs_args: {n_rel_samples: 5, p_rel: -2.0, rel_tau: 1.0e-5, abs_tau: 1.0e-5, w_v: 0.8} - - name: DetKUCB ($p_{rel}=-2.0, w_v=0.8, \tau_r=10^{-5}, \tau_a=0.1$) (rel, abs) - model_type: MaxVar - obs_args: {n_rel_samples: 5, p_rel: -2.0, rel_tau: 0.1, abs_tau: 1.0e-5, w_v: 0.8} - - name: DetKUCB ($p_{rel}=-2.0, w_v=0.8, \tau_r=0.1, \tau_a=10^{-5}$) (rel, abs) - model_type: MaxVar - obs_args: {n_rel_samples: 5, p_rel: -2.0, rel_tau: 0.1, abs_tau: 1.0e-5, w_v: 0.8} - - name: DetKUCB ($p_{rel}=-5.0, w_v=0.8, \tau=0.1$) (rel, abs) - model_type: MaxVar - obs_args: {n_rel_samples: 5, p_rel: -5.0, rel_tau: 0.1, abs_tau: 0.1, w_v: 0.8} - - name: DetKUCB ($p_{rel}=-5.0, w_v=0.8, \tau=10^{-5}$) (rel, abs) - model_type: MaxVar - obs_args: {n_rel_samples: 5, p_rel: -5.0, rel_tau: 1.0e-5, abs_tau: 1.0e-5, w_v: 0.8} - - name: DetKUCB ($p_{rel}=-5.0, w_v=0.8, \tau_r=10^{-5}, \tau_a=0.1$) (rel, abs) - model_type: MaxVar - obs_args: {n_rel_samples: 5, p_rel: -5.0, rel_tau: 0.1, abs_tau: 1.0e-5, w_v: 0.8} - - name: DetKUCB ($p_{rel}=-5.0, w_v=0.8, \tau_r=0.1, \tau_a=10^{-5}$) (rel, abs) - model_type: MaxVar - obs_args: {n_rel_samples: 5, p_rel: -5.0, rel_tau: 0.1, abs_tau: 1.0e-5, w_v: 0.8} - - name: DetKUCB ($p_{rel}=-10.0, w_v=0.8, \tau=0.1$) (rel, abs) - model_type: MaxVar - obs_args: {n_rel_samples: 5, p_rel: -10.0, rel_tau: 0.1, abs_tau: 0.1, w_v: 0.8} - - name: DetKUCB ($p_{rel}=-10.0, w_v=0.8, \tau=10^{-5}$) (rel, abs) - model_type: MaxVar - obs_args: {n_rel_samples: 5, p_rel: -10.0, rel_tau: 1.0e-5, abs_tau: 1.0e-5, w_v: 0.8} - - name: DetKUCB ($p_{rel}=-10.0, w_v=0.8, \tau_r=10^{-5}, \tau_a=0.1$) (rel, abs) - model_type: MaxVar - obs_args: {n_rel_samples: 5, p_rel: -10.0, rel_tau: 0.1, abs_tau: 1.0e-5, w_v: 0.8} - - name: DetKUCB ($p_{rel}=-10.0, w_v=0.8, \tau_r=0.1, \tau_a=10^{-5}$) (rel, abs) - model_type: MaxVar - obs_args: {n_rel_samples: 5, p_rel: -10.0, rel_tau: 0.1, abs_tau: 1.0e-5, w_v: 0.8} + obs_args: {n_rel_samples: 5, p_rel: -1.0, rel_tau: 0.1, abs_tau: 0.05, w_v: 0.75, selector: entropy} diff --git a/ordinal_likelihoods.py b/ordinal_likelihoods.py index f2b4623..deaceec 100644 --- a/ordinal_likelihoods.py +++ b/ordinal_likelihoods.py @@ -2,6 +2,7 @@ import matplotlib.pyplot as plt import GPpref import nice_plot_colors as npc +import plot_tools from scipy.stats import norm plt.rc('font',**{'family':'serif','sans-serif':['Computer Modern Roman']}) plt.rc('text', usetex=True) @@ -17,7 +18,7 @@ def beta_discrete(b, f, n_cat=5): n_ords = 5 n_f = 101 -f = np.linspace(-9.0, 9.0, n_f) +f = np.linspace(-3.0, 3.0, n_f) # Integrated beta sigma = 1.0 @@ -31,13 +32,14 @@ def beta_discrete(b, f, n_cat=5): beta_lines = [] for i, p in enumerate(Py.T): beta_lines.extend(ah.plot(f, p, color=npc.lines[i])) -ah.legend(beta_lines, ['$p(y={0}|f)$'.format(ni) for ni in range(n_ords)]) +ah.legend(beta_lines, ['$p(y={0}|f)$'.format(ni+1) for ni in range(n_ords)]) # Logistic ordinal -log_sigma = 0.5 -b = np.linspace(-1.1, 1.1, n_ords-1) # Log breakpoints +ord_sigma = 0.25 +ord_b = 1.2 +b = np.linspace(-ord_b, ord_b, n_ords-1) # Log breakpoints Py_ord = np.zeros((n_f, n_ords), dtype='float') -z = lambda ff, bb: (bb-ff)/log_sigma +z = lambda ff, bb: (bb-ff) / ord_sigma Py_ord[:, 0] = norm.cdf(z(f, b[0])) Py_ord[:, -1] = 1.0-norm.cdf(z(f, b[-1])) for i in range(1, n_ords-1): @@ -51,8 +53,9 @@ def beta_discrete(b, f, n_cat=5): ah.set_ylabel('$p(y|f)$') -# Testing the Ordinal Probit class -orpro = GPpref.OrdinalProbit(log_sigma, b[-1], n_ords, eps=1.0e-5) +# Testing the Ordinal Probit class in GPPref +ord_kwargs = {'sigma': ord_sigma, 'b': ord_b, 'n_ordinals': n_ords, 'eps': 1.0e-5} +orpro = GPpref.OrdinalProbit(**ord_kwargs) Py_c = np.zeros((n_f, n_ords), dtype='float') dPy_c = np.zeros((n_f, n_ords), dtype='float') d2Py_c = np.zeros((n_f, n_ords), dtype='float') @@ -62,14 +65,63 @@ def beta_discrete(b, f, n_cat=5): d2p, dp, p = orpro.derivatives(y*(i+1), fv) d2Py_c[:,i], dPy_c[:,i], Py_c[:,i] = d2p.diagonal(), dp.flat, p.flat -fh2, ah2 = plt.subplots(1, 3) +fh2, ah2 = plt.subplots(2, 2) +ah2 = ah2.flatten() ah2[0].plot(f, np.exp(Py_c)) ah2[0].set_title('$P(y|f(x))$') -ah2[1].plot(f, dPy_c) -ah2[1].set_title('$\partial l(y,f(x)) / \partial f(x)$') -ah2[2].plot(f, d2Py_c) -ah2[2].set_title('$\partial^2 l(y,f(x)) / \partial f(x)^2$') - +ah2[0].set_ylabel('$P(y)$') + +ah2[1].plot(f, Py_c) +ah2[1].set_title('$ln P(y|f(x))$') +ah2[1].set_ylabel('$ln P(y)$') + +ah2[2].plot(f, dPy_c) +ah2[2].set_title('$\partial ln P(y|f(x) / \partial f(x)$') +ah2[2].set_ylabel('$\partial ln P(y) / \partial f(x)$') + +ah2[3].plot(f, d2Py_c) +ah2[3].set_title('$\partial^2 l(y,f(x)) / \partial f(x)^2$') +ah2[3].set_ylabel('$\partial^2 ln P(y) / \partial f(x)^2$') +for a in ah2: + a.set_xlabel('$f(x)$') + + +# Test the sampler +def fun(x): + return x + # return 2.5*np.sin(x/2.0) +obs_fun = GPpref.AbsObservationSampler(fun, 'OrdinalProbit', ord_kwargs) +x_plot = f +x_test = np.atleast_2d(x_plot).T +f_true = obs_fun.f(x_test) +y_samples = np.atleast_2d(np.arange(1, n_ords+1, 1, dtype='int')).T +p_y_array = obs_fun.observation_likelihood_array(x_test, y_samples) +z, mu = obs_fun.generate_observations(x_test) + +fh3, ah3 = plt.subplots(1, 2) +plot_tools.plot_with_bounds(ah3[0], x_test, f_true, ord_sigma, c=npc.lines[0]) +abs_extent = [x_test[0, 0], x_test[-1, 0], y_samples[0, 0]-0.5, y_samples[-1, 0]+0.5] +h_pat = ah3[1].imshow(p_y_array, origin='lower', extent=abs_extent, aspect='auto') +ah3[1].plot(x_test, z, 'w+') +h_yt, = ah3[1].plot(x_test, mu, c=npc.lines[1]) +ah3[1].legend([h_yt], ['$E[y]$']) +fh3.colorbar(h_pat, ax=ah3[1]) + + +# Plot some beta likelihood stuff for Thane +# beta_likelihood = GPpref.AbsBoundProbit(sigma=1.0, v=25.0) +# y = np.atleast_2d(np.linspace(0.0, 1.0, 201)).T +# f = np.linspace(-2.0, 2.0, 5) +# p_y = np.zeros((y.shape[0], len(f)), dtype='float') +# for i, fxi in enumerate(f): +# p_y[:, i:i + 1] = beta_likelihood.likelihood(y, fxi) +# fh4, ah4 = plt.subplots() +# h_b = plt.plot(y, p_y) +# ah4.legend(h_b, ['$f = {0:0.1f}$'.format(fi) for fi in f]) +# ah4.set_ylim([0, 10.0]) +# ah4.set_xlabel('$y$') +# ah4.set_ylabel('$p(y|f)$') +# ah4.set_title('Symmetric beta likelihood') plt.show() diff --git a/plot_tools.py b/plot_tools.py index 3a0e662..2726cf6 100644 --- a/plot_tools.py +++ b/plot_tools.py @@ -48,7 +48,7 @@ def plot_setup_rel(t_l = r'Latent function, $f(x)$', t_r = r'Relative likelihood ax_r.set_ylabel('$x_1$') return fig, (ax_l, ax_r) -def plot_setup_2d(t_l = r'Latent function, $f(x)$', t_a = r'Absolute likelihood, $p(u | f(x))$', +def plot_setup_2d(t_l = r'Latent function, $f(x)$', t_a = r'Absolute likelihood, $p(y | f(x))$', t_r = r'Relative likelihood, $P(x_0 \succ x_1 | f(x_0), f(x_1))$'): fig, (ax_l, ax_a, ax_r) = plt.subplots(1, 3) @@ -62,7 +62,7 @@ def plot_setup_2d(t_l = r'Latent function, $f(x)$', t_a = r'Absolute likelihood, # Absolute likelihood ax_a.set_title(t_a) ax_a.set_xlabel('$x$') - ax_a.set_ylabel('$u$') + ax_a.set_ylabel('$y$') # Relative likelihood ax_r.set_title(t_r) @@ -78,8 +78,7 @@ def plot_relative_likelihood(ax, p_y, extent): ax.get_figure().colorbar(h_p, ax=ax) return h_p - -def true_plots(xt, ft, mu_t, rel_sigma, y_samples, p_a_y, p_r_y, xa_train, ya_train, uvr_train, fuvr_train, yr_train, +def true_plots(xt, ft, mu_t, rel_sigma, y_samples, p_a_y, p_r_y, xa_train=None, ya_train=None, uvr_train=None, fuvr_train=None, yr_train=None, class_icons=['ko', 'wo'], marker_options={'mec':'k', 'mew':0.5}, *args, **kwargs): @@ -90,18 +89,20 @@ def true_plots(xt, ft, mu_t, rel_sigma, y_samples, p_a_y, p_r_y, xa_train, ya_tr plot_with_bounds(ax_l, xt, ft, rel_sigma, c=lines[0]) # True absolute likelihood - abs_extent = [xt[0, 0], xt[-1, 0], y_samples[0, 0], y_samples[-1, 0]] - h_pat = ax_a.imshow(p_a_y, origin='lower', extent=abs_extent) - if xa_train.shape[0] > 0: + dely = y_samples[1, 0] - y_samples[0, 0] + abs_extent = [xt[0, 0], xt[-1, 0], y_samples[0, 0] - 0.5*dely, y_samples[-1, 0] + 0.5*dely] + h_pat = ax_a.imshow(p_a_y, origin='lower', extent=abs_extent, aspect='auto') + if xa_train is not None and xa_train.shape[0] > 0: ax_a.plot(xa_train, ya_train, 'w+') h_yt, = ax_a.plot(xt, mu_t, c=lines[0]) - ax_a.legend([h_yt], ['$E[u]$']) + ax_a.legend([h_yt], ['$E[y]$']) + ax_a.set_xlim(xt[0], xt[-1]) fig.colorbar(h_pat, ax=ax_a) # True relative likelihood rel_y_extent = [xt[0, 0], xt[-1, 0], xt[0, 0], xt[-1, 0]] h_prt = plot_relative_likelihood(ax_r, p_r_y, extent=rel_y_extent) - if xt.shape[0] > 0: + if uvr_train is not None and xt.shape[0] > 0: for uv, fuv, y in zip(uvr_train, fuvr_train, yr_train): ax_r.plot(uv[0], uv[1], class_icons[(y[0] + 1) / 2], **marker_options) ax_l.plot(uv, fuv, 'b-', color=lighten(lines[0])) @@ -149,14 +150,16 @@ def estimate_plots(xt, ft, mu_t, fhat, vhat, E_y, rel_sigma, ax_l.legend([hf, hf_hat], [r'True latent function, $f(x)$', r'$\mathcal{GP}$ estimate $\hat{f}(x)$']) # Absolute posterior likelihood - abs_extent = [xt[0, 0], xt[-1, 0], y_samples[0, 0], y_samples[-1, 0]] - h_pap = ax_a.imshow(p_a_y, origin='lower', extent=abs_extent) + dely = y_samples[1, 0]-y_samples[0, 0] + abs_extent = [xt[0, 0], xt[-1, 0], y_samples[0, 0]-0.5*dely, y_samples[-1, 0]+0.5*dely] + h_pap = ax_a.imshow(p_a_y, origin='lower', extent=abs_extent, aspect='auto') h_yt, = ax_a.plot(xt, mu_t, c=lines[0]) hEy, = ax_a.plot(xt, E_y, color=lines[3]) if xa_train.shape[0] > 0: ax_a.plot(xa_train, ya_train, 'w+') + ax_a.set_xlim(xt[0], xt[-1]) ax_a.legend([h_yt, hEy], - [r'True mean, $E[u]$', r'Posterior mean, $E_{p(u|\mathcal{Y})}\left[u\right]$']) + [r'True mean, $E[y]$', r'Posterior mean, $E_{p(y|\mathcal{Y})}\left[y\right]$']) fig.colorbar(h_pap, ax=ax_a) # Relative posterior likelihood diff --git a/pref_active_learning.py b/pref_active_learning.py index 3dc52c3..02102da 100644 --- a/pref_active_learning.py +++ b/pref_active_learning.py @@ -15,26 +15,21 @@ save_plots = False -with open('./data/statruns2_oct2017.yaml', 'rt') as fh: +with open('./data/ordinal_test.yaml', 'rt') as fh: wave = yaml.safe_load(fh) try: np.random.seed(wave['statrun_params']['randseed']) + n_rel_train, n_abs_train = wave['statrun_params']['n_rel_train'], wave['statrun_params']['n_abs_train'] + n_queries = wave['statrun_params']['n_queries'] except KeyError: np.random.seed(0) - -n_rel_train = 0 -n_abs_train = 1 -n_queries = 20 - -delta_f = 1e-5 + n_rel_train = 1 + n_abs_train = 0 + n_queries = 20 n_xplot = 101 -n_mcsamples = 1000 -n_ysamples = 101 - keep_f = True -verbose = 1 log_hyp = np.log(wave['hyperparameters']) @@ -48,12 +43,10 @@ fig_dir = 'fig/' + nowstr + '/' ptt.ensure_dir(fig_dir) print "Figures will be saved to: {0}".format(fig_dir) - with open(fig_dir+'params.yaml', 'wt') as fh: - yaml.safe_dump(wave, fh) pdf_pages = PdfPages(fig_dir+'posterior_all.pdf') -rel_obs_fun = GPpref.RelObservationSampler(true_function, GPpref.PrefProbit(**wave['rel_obs_params'])) -abs_obs_fun = GPpref.AbsObservationSampler(true_function, GPpref.AbsBoundProbit(**wave['abs_obs_params'])) +rel_obs_fun = GPpref.RelObservationSampler(true_function, wave['GP_params']['rel_likelihood'], wave['rel_obs_params']) +abs_obs_fun = GPpref.AbsObservationSampler(true_function, wave['GP_params']['abs_likelihood'], wave['abs_obs_params']) rel_sigma = wave['rel_obs_params']['sigma'] # True function @@ -61,14 +54,14 @@ x_test = np.atleast_2d(x_plot).T f_true = abs_obs_fun.f(x_test) mu_true = abs_obs_fun.mean_link(x_test) -mc_samples = np.random.normal(size=n_mcsamples) -abs_y_samples = np.atleast_2d(np.linspace(0.01, 0.99, n_ysamples)).T +abs_y_samples = abs_obs_fun.l.y_list p_abs_y_true = abs_obs_fun.observation_likelihood_array(x_test, abs_y_samples) p_rel_y_true = rel_obs_fun.observation_likelihood_array(x_test) -# Training data -x_rel, uvi_rel, uv_rel, y_rel, fuv_rel = rel_obs_fun.generate_n_observations(n_rel_train, n_xdim=1) -x_abs, y_abs, mu_abs = abs_obs_fun.generate_n_observations(n_abs_train, n_xdim=1) +# Training data - note the shifted domain to get the sample out of the way +far_domain = np.array([[-3.0], [-2.0]]) +x_rel, uvi_rel, uv_rel, y_rel, fuv_rel = rel_obs_fun.generate_n_observations(n_rel_train, n_xdim=1, domain=far_domain) +x_abs, y_abs, mu_abs = abs_obs_fun.generate_n_observations(n_abs_train, n_xdim=1, domain=far_domain) # Plot true functions @@ -80,12 +73,15 @@ fig_t.savefig(fig_dir+'true.pdf', bbox_inches='tight') # Construct active learner object -GP_kwargs = {'x_rel':x_rel, 'uvi_rel':uvi_rel, 'x_abs':x_abs, 'y_rel':y_rel, 'y_abs':y_abs, 'delta_f':delta_f, - 'rel_likelihood':GPpref.PrefProbit(), 'abs_likelihood':GPpref.AbsBoundProbit()} -learner_kwargs = wave['learners'][0] + +# Construct GP object +prefGP = GPpref.PreferenceGaussianProcess(x_rel, uvi_rel, x_abs, y_rel, y_abs, **wave['GP_params']) + +model_kwargs = {'x_rel':x_rel, 'uvi_rel':uvi_rel, 'x_abs':x_abs, 'y_rel':y_rel, 'y_abs':y_abs} +model_kwargs.update(wave['GP_params']) +learner_kwargs = wave['learners'][-1] learner = active_learners.Learner(**learner_kwargs) -learner.build_model(GP_kwargs) -learner.model.set_hyperparameters(log_hyp) +learner.build_model(model_kwargs) # Get initial solution learner.model.set_hyperparameters(log_hyp) @@ -93,31 +89,39 @@ learner.model.print_hyperparameters() fig_p, (ax_p_l, ax_p_a, ax_p_r) = learner.model.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, - abs_y_samples, mc_samples) + abs_y_samples) if save_plots: + wave['learners'] = [learner_kwargs] + with open(fig_dir+'params.yaml', 'wt') as fh: + yaml.safe_dump(wave, fh) pdf_pages.savefig(fig_p, bbox_inches='tight') # fig_p.savefig(fig_dir+'posterior00.pdf', bbox_inches='tight') +print learner_kwargs for obs_num in range(n_queries): + if learner.update_p_rel: + linear_p_rel = max(0.0, (n_queries - obs_num) / float(n_queries)) + learner.obs_arguments['p_rel'] = linear_p_rel # if 'p_rel' in obs_arguments: # obs_arguments['p_rel'] = max(0.0, (n_queries - obs_num) / float(n_queries)) + t0 = time.time() next_x = learner.select_observation() if next_x.shape[0] == 1: next_y, next_f = abs_obs_fun.generate_observations(next_x) learner.model.add_observations(next_x, next_y, keep_f=keep_f) - print 'Abs: x:{0}, y:{1}'.format(next_x[0], next_y[0]) + print 'Abs: x:{0}, y:{1}'.format(next_x[0], next_y[0]), else: next_y, next_uvi, next_fx = rel_obs_fun.gaussian_multi_pairwise_sampler(next_x) next_fuv = next_fx[next_uvi][:, :, 0] fuv_rel = np.concatenate((fuv_rel, next_fuv), 0) learner.model.add_observations(next_x, next_y, next_uvi, keep_f=keep_f) - print 'Rel: x:{0}, best_index:{1}'.format(next_x.flatten(), next_uvi[0, 1]) - + print '{n:02} - Rel: x:{0}, best_index:{1}'.format(next_x.flatten(), next_uvi[0, 1], n=obs_num+1), + print 't = {0:0.3f}s'.format(time.time()-t0) f = learner.model.solve_laplace() if save_plots: fig_p, (ax_p_l, ax_p_a, ax_p_r) = learner.model.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, - abs_y_samples, mc_samples) + abs_y_samples) pdf_pages.savefig(fig_p, bbox_inches='tight') # fig_p.savefig(fig_dir+'posterior{0:02d}.pdf'.format(obs_num+1), bbox_inches='tight') plt.close(fig_p) @@ -128,7 +132,7 @@ pdf_pages.close() else: fig_post, (ax_p_l, ax_p_a, ax_p_r) = learner.model.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, - abs_y_samples, mc_samples) + abs_y_samples) plt.show() print "Finished!" From 43862fb3c7e79816f7b64bbfbfceae6c0e345cba Mon Sep 17 00:00:00 2001 From: Nicholas Lawrance Date: Fri, 1 Dec 2017 16:08:09 -0800 Subject: [PATCH 20/38] Lots of updates - unspecified. --- GP_preference_demo.py | 2 +- GPpref.py | 12 +-- active_learners.py | 109 ++++++++++++++------------- active_statruns.py | 34 ++++++--- plot_statruns.py | 158 +++++++++++++++++++++++++++++++++++----- plot_tools.py | 4 +- pref_active_learning.py | 8 +- test_data.py | 16 +++- 8 files changed, 247 insertions(+), 96 deletions(-) diff --git a/GP_preference_demo.py b/GP_preference_demo.py index 0054b22..88345d6 100644 --- a/GP_preference_demo.py +++ b/GP_preference_demo.py @@ -14,7 +14,7 @@ use_test_data = False # test_data.data3 # verbose = 2 -with open('./data/ordinal_test.yaml', 'rt') as fh: +with open('./data/statruns_dec2017.yaml', 'rt') as fh: wave = yaml.safe_load(fh) try: diff --git a/GPpref.py b/GPpref.py index 1dbfce8..1c4bd3b 100644 --- a/GPpref.py +++ b/GPpref.py @@ -409,7 +409,7 @@ def generate_samples(self, f): class PreferenceGaussianProcess(object): def __init__(self, x_rel, uvi_rel, x_abs, y_rel, y_abs, rel_likelihood='PrefProbit', delta_f=1.0e-5, - abs_likelihood='AbsBoundProbit', verbose=False, hyper_counts=[2, 1, 2]): + abs_likelihood='AbsBoundProbit', verbose=False, hyper_counts=[2, 1, 2], rel_kwargs={}, abs_kwargs={}): # hyperparameters are split by hyper_counts, where hyper_counts should contain three integers > 0, the first is # the number of hypers in the GP covariance, second is the number in the relative likelihood, last is the number # in the absolute likelihood. hyper_counts.sum() should be equal to len(log_hyp) @@ -419,8 +419,8 @@ def __init__(self, x_rel, uvi_rel, x_abs, y_rel, y_abs, rel_likelihood='PrefProb self.set_observations(x_rel, uvi_rel, x_abs, y_rel, y_abs) self.delta_f = delta_f - self.rel_likelihood = getattr(sys.modules[__name__], rel_likelihood)() # This calls the constructor from string - self.abs_likelihood = getattr(sys.modules[__name__], abs_likelihood)() + self.rel_likelihood = getattr(sys.modules[__name__], rel_likelihood)(**rel_kwargs) # This calls the constructor from string + self.abs_likelihood = getattr(sys.modules[__name__], abs_likelihood)(**abs_kwargs) self.verbose = verbose @@ -687,8 +687,10 @@ def gaussian_multi_pairwise_sampler(self, x): return y, uvi, fx class AbsObservationSampler(ObservationSampler): - def observation_likelihood_array(self, x, y): + def observation_likelihood_array(self, x, y=None): fx = self.f(x) + if y is None: + y = self.l.y_list p_y = np.zeros((y.shape[0], fx.shape[0]), dtype='float') for i, fxi in enumerate(fx): p_y[:, i:i + 1] = self.l.likelihood(y, fxi[0]) @@ -703,8 +705,6 @@ def generate_n_observations(self, n, n_xdim=1, domain=None): y, mu = self.generate_observations(x) return x, y, mu - def get_y_samples(self): - return self.l.get_y_samples() class RelObservationSampler(ObservationSampler): def observation_likelihood_array(self, x, y=-1): diff --git a/active_learners.py b/active_learners.py index c0a37aa..909d339 100644 --- a/active_learners.py +++ b/active_learners.py @@ -593,98 +593,105 @@ def calculate_threshold_utility(self, fhat, vhat, n_samples, y_threshold): class OrdinalSampler(PeakComparitor): - def model_value(self, fhat, vhat): - # Probability of missclassification - p_y, mu = self.abs_likelihood.posterior_likelihood(fhat, vhat) - - pass - - def test_observation(self, x, y, uvi, x_test, n_samples, y_threshold, f = None): + def model_value(self, fhat, vhat, y_threshold): + # Probability of correct classification + p_y, mu = self.abs_likelihood.posterior_likelihood(fhat, vhat) + selection = np.argmax(p_y, axis=0)+1 + value = 0.0 + for i in range(len(fhat)): + if selection[i] >= y_threshold: + value += p_y[y_threshold-1:,i].sum() + else: + value += p_y[0:y_threshold-1,i].sum() + return value/len(fhat) + + def test_observation(self, x, y, uvi, x_test, y_threshold): self.add_observations(x, y, uvi, keep_f=True) self.solve_laplace() fhat, vhat = self.predict_latent(x_test) - util = self.model_value(fhat, vhat) + util = self.model_value(fhat, vhat, y_threshold) self.reset_observations() return util - def select_observation(self, domain=None, x_test=None, n_test=50, n_samples=50, y_threshold=0.8, p_pref_tol=1e-3, n_mc_abs=5): - # n_test is the number of test point locations on the input function - # n_samples is the number of functions sampled from the posterior for estimating the utility - # n_mc_abs is the number of proposed observations sampled from the current posterior for absolute estimates + def get_rating_set_uv(self, n_ratings): + y = np.ones((n_ratings - 1, 1), dtype='int') + uv = [] + for i in range(n_ratings): + other_i = np.delete(np.arange(n_ratings), i) + uv.append(np.hstack((np.atleast_2d(other_i).T, y*i))) + return y, uv + + + def select_observation(self, domain=None, x_test=None, n_test=50, y_threshold=5, p_pref_tol=1e-3, n_rel_samples=2, n_mc_samples=100): # Generate a set of test points in the domain (if not specified) if x_test is None: # x_test = self.linear_domain_sampler(n_test, domain) x_test = self.uniform_domain_sampler(n_test, domain) - # x_test.sort(axis=0) n_test = len(x_test) - # Sample a set of functions from the current posterior # We save the f value because otherwise it gets out of whack when we add observations flap = self.solve_laplace() fhat, vhat = self.predict_latent(x_test) - # base_utility = self.calculate_threshold_utility(fhat, vhat, n_samples, y_threshold) - # Check a random set of pairwise relatives and all absolutes (massive sampling) + # Sample a set of functions from the current posterior + f_post = np.random.multivariate_normal(fhat.flatten(), vhat, n_mc_samples) - # Generate a set of random pairs (this randomly pairs all x_test points) - uvi = np.random.choice(n_test, (n_test/2, 2), replace=False) - p_pref = self.rel_likelihood.posterior_likelihood(fhat, vhat, uvi, y=-1) + # Generate a set of random groups of n_rel_samples from the input space + uvi = np.array([np.random.choice(n_test, n_rel_samples, replace=False) for i in range(n_test)]) V_max_rel = 0.0 self.store_observations() # Relative observations t_rel = time.time() - # Now calculate the expected value for each observation pair + y_uv, full_uv = self.get_rating_set_uv(n_rel_samples) + + # Calculate the expected value for each observation set for i, uv in enumerate(uvi): x = x_test[uv] + p_pref = np.zeros(n_rel_samples) + for fp in f_post: + p_pref[np.argmax(fp[uv])] += 1 + p_pref /= n_mc_samples V_rel = 0.0 - try: - if p_pref[i] < p_pref_tol: - V_rel += (1-p_pref[i])*self.test_observation(x, self._plus_y_obs, self._default_uvi, x_test, n_samples, y_threshold, f=flap) - elif p_pref[i] > 1.0-p_pref_tol: - V_rel += p_pref[i]*self.test_observation(x, self._minus_y_obs, self._default_uvi, x_test, n_samples, y_threshold, f=flap) - else: - V_rel += (1-p_pref[i])*self.test_observation(x, self._plus_y_obs, self._default_uvi, x_test, n_samples, y_threshold, f=flap) - V_rel += p_pref[i] * self.test_observation(x, self._minus_y_obs, self._default_uvi, x_test, n_samples, y_threshold, f=flap) - if V_rel >= V_max_rel: - V_max_rel = V_rel - x_best_rel = x - except GPpref.LaplaceException as exc: - print "Failed in relative test observation, x = [{0}, {1}]".format(x[0], x[1]) - raise exc + + for i, p_x in enumerate(p_pref): + if p_x > p_pref_tol: + V_rel += p_x*self.test_observation(x, y_uv, full_uv[i], x_test, y_threshold) + if V_rel >= V_max_rel: + V_max_rel = V_rel + x_best_rel = x # best_n = np.argpartition(V, -n_comparators)[-n_comparators:] # best = np.argmax(V) - if self.verbose: - print 'V_max_rel = {0}, x = {2}, t = {1}s'.format(V_max_rel[0], time.time()-t_rel, x_best_rel[:,0]) + if self.verbose >= 1: + print 'V_max_rel = {0}, x = {2}, t = {1}s'.format(V_max_rel, time.time()-t_rel, x_best_rel[:,0]) # Absolute queries V_max = 0.0 t_rel = time.time() - for i, x in enumerate(x_test): - F = fhat[i] + np.random.randn(n_mc_abs)*np.sqrt(vhat[i,i]) - Y, mu = self.abs_likelihood.generate_samples(F) + p_y, mu = self.abs_likelihood.posterior_likelihood(fhat, vhat) + for x, ppy in zip(x_test, p_y.T): V_abs = 0.0 - Y = np.clip(Y, 1e-2, 1-1e-2) # I had stability problems in Laplace with values approaching 0 or 1 - for y in Y: - try: - V_abs += self.test_observation(x, y, None, x_test, n_samples, y_threshold, f=flap) - except ValueError: - print "NaN Issue" - except GPpref.LaplaceException as exc: - print "Failed in absolute test observation, x = {0}, y = {1}".format(x, y) - raise exc - V_abs /= n_mc_abs + for y_obs, p_obs in enumerate(ppy): + if p_obs > p_pref_tol: + V_abs += p_obs*self.test_observation(x, y_obs+1, None, x_test, y_threshold) if V_abs > V_max: V_max = V_abs x_best = x - if self.verbose: - print 'V_max_abs = {0}, x = {2}, t = {1}s'.format(V_max, time.time() - t_rel, x_best) + if self.verbose >= 1: + print 'V_max_abs = {0}, t = {1}s'.format(V_max, time.time() - t_rel) if V_max_rel > V_max: x_best = x_best_rel return x_best + +class OrdinalOptimist(OrdinalSampler): + def model_value(self, fhat, vhat, y_threshold): + # Probability of values above threshold + p_y, mu = self.abs_likelihood.posterior_likelihood(fhat, vhat) + value = p_y[y_threshold-1:,:].sum()/len(fhat) + return value/len(fhat) diff --git a/active_statruns.py b/active_statruns.py index f8a97d8..f3e52dc 100644 --- a/active_statruns.py +++ b/active_statruns.py @@ -9,6 +9,8 @@ import yaml np.set_printoptions(precision=3) +wrms_fun = test_data.wrms_misclass +wrms_args = {'w_power': 1} now_time = time.strftime("%Y_%m_%d-%H_%M") @@ -43,18 +45,21 @@ # True function x_plot = np.linspace(0.0, 1.0, statrun_params['n_xtest'], dtype='float') x_test = np.atleast_2d(x_plot).T +far_domain = np.array([[-3.0], [-2.0]]) # Construct active learner objects n_learners = len(run_parameters['learners']) learners = [] names = [] obs_array = [] +full_obs = {} for l in run_parameters['learners']: if 'n_rel_samples' in l['obs_args']: l['obs_args']['n_rel_samples'] = n_rel_samples learners.append(active_learners.Learner(**l)) names.append(l['name']) obs_array.append({'name': l['name'], 'obs': []}) + full_obs[l['name']] = [None] * n_trials wrms_results = np.zeros((n_learners, n_queries+1, n_trials)) true_pos_results = np.zeros((n_learners, n_queries+1, n_trials), dtype='int') @@ -86,9 +91,12 @@ p_rel_y_true = rel_obs_fun.observation_likelihood_array(x_test) # Initial data - x_rel, uvi_rel, uv_rel, y_rel, fuv_rel = rel_obs_fun.generate_n_observations(statrun_params['n_rel_train'], n_xdim=1) - x_abs, y_abs, mu_abs = abs_obs_fun.generate_n_observations(statrun_params['n_abs_train'], n_xdim=1) - model_kwargs = {'x_rel': x_rel, 'uvi_rel': uvi_rel, 'x_abs': x_abs, 'y_rel': y_rel, 'y_abs': y_abs} + x_rel, uvi_rel, uv_rel, y_rel, fuv_rel = rel_obs_fun.generate_n_observations(statrun_params['n_rel_train'], + n_xdim=1, domain=far_domain) + x_abs, y_abs, mu_abs = abs_obs_fun.generate_n_observations(statrun_params['n_abs_train'], n_xdim=1, + domain=far_domain) + model_kwargs = {'x_rel': x_rel, 'uvi_rel': uvi_rel, 'x_abs': x_abs, 'y_rel': y_rel, 'y_abs': y_abs, + 'rel_kwargs': run_parameters['rel_obs_params'], 'abs_kwargs': run_parameters['abs_obs_params']} model_kwargs.update(run_parameters['GP_params']) # Get initial solution @@ -100,13 +108,16 @@ y_abs_est = learner.model.abs_posterior_mean(x_test, fhat, vhat) best_points_est = set(np.argpartition(y_abs_est.flatten(), -n_best_points)[-n_best_points:]) - wrms_results[nl, 0, trial_number] = test_data.wrms(y_abs_true, y_abs_est) + wrms_results[nl, 0, trial_number] = wrms_fun(y_abs_true, y_abs_est, **wrms_args) true_pos_results[nl, 0, trial_number] = len(best_points_set.intersection(best_points_est)) selected_error[nl, 0, trial_number] = test_data.wrms(y_abs_true[best_points], y_abs_est[best_points], weight=False) if calc_relative_error: p_rel_y_post = learner.model.rel_posterior_likelihood_array(fhat=fhat, varhat=vhat) relative_error[nl, 0, trial_number] = test_data.rel_error(y_abs_true, p_rel_y_true, y_abs_est, p_rel_y_post, weight=True) + obs_tuple = learner.model.get_observations() + full_obs[learner.name][trial_number] = [test_data.ObsObject(*obs_tuple)] + for obs_num in range(n_queries): t0 = time.time() linear_p_rel = max(0.0, (n_queries-obs_num)/float(n_queries)) @@ -117,23 +128,24 @@ learner.obs_arguments['p_rel'] = linear_p_rel next_x = learner.model.select_observation(**learner.obs_arguments) + next_uvi = None if next_x.shape[0] == 1: next_y, next_f = abs_obs_fun.generate_observations(next_x) - learner.model.add_observations(next_x, next_y) # print 'Abs: x:{0}, y:{1}'.format(next_x[0], next_y[0]) else: next_y, next_uvi, next_fx = rel_obs_fun.gaussian_multi_pairwise_sampler(next_x) next_fuv = next_fx[next_uvi][:,:,0] fuv_rel = np.concatenate((fuv_rel, next_fuv), 0) - learner.model.add_observations(next_x, next_y, next_uvi) # print 'Rel: x:{0}, best_index:{1}'.format(next_x.flatten(), next_uvi[0, 1]) + full_obs[learner.name][trial_number].append((next_x, next_y, next_uvi)) + learner.model.add_observations(next_x, next_y, next_uvi) f = learner.model.solve_laplace() fhat, vhat = learner.model.predict_latent(x_test) y_abs_est = learner.model.abs_posterior_mean(x_test, fhat, vhat) # Get selected best point set and error results best_points_est = set(np.argpartition(y_abs_est.flatten(), -n_best_points)[-n_best_points:]) - wrms_results[nl, obs_num+1, trial_number] = test_data.wrms_misclass(y_abs_true, y_abs_est) + wrms_results[nl, obs_num+1, trial_number] = wrms_fun(y_abs_true, y_abs_est, **wrms_args) true_pos_results[nl, obs_num+1, trial_number] = len(best_points_set.intersection(best_points_est)) selected_error[nl, obs_num+1, trial_number] = test_data.wrms(y_abs_true[best_points], y_abs_est[best_points], weight=False) if calc_relative_error: @@ -155,11 +167,11 @@ trial_number += 1 if relative_error is not None: - plot_statruns.save_data(data_dir, wrms_results[:,:,:trial_number], true_pos_results[:, :, :trial_number], - selected_error[:, :, :trial_number], obs_array, relative_error[:, :, :trial_number]) + plot_statruns.save_data(data_dir, wrms_results, true_pos_results, selected_error, obs_array, full_obs=full_obs, + rel_err=relative_error, t=trial_number) else: - plot_statruns.save_data(data_dir, wrms_results[:,:,:trial_number], true_pos_results[:, :, :trial_number], - selected_error[:, :, :trial_number], obs_array) + plot_statruns.save_data(data_dir, wrms_results, true_pos_results, selected_error, obs_array, full_obs=full_obs, + t=trial_number) waver.save(data_dir+'wave_data.pkl') diff --git a/plot_statruns.py b/plot_statruns.py index 70f6be5..5480e86 100644 --- a/plot_statruns.py +++ b/plot_statruns.py @@ -1,6 +1,10 @@ import numpy as np import matplotlib.pyplot as plt import pickle +import yaml +import test_data +import GPpref +import active_learners from Tkinter import Tk from tkFileDialog import askdirectory # from test_data import ObsObject @@ -54,9 +58,9 @@ def plot_results(wrms_results, true_pos_results, selected_error, obs_array, rela selected_error=selected_error[methods_indexes,:,:] - f0, ax0 = single_plot(wrms_results, names=names, ylabel='Weighted RMSE', bars=bars, percentile=-1) - f1, ax1 = single_plot(true_pos_results, names=names, ylabel='True positive selections (out of 30)', bars=True, precut=1, percentile=-1) - f2, ax2 = single_plot(selected_error, names=names, ylabel='RMSE of best paths', bars=True, precut=1, percentile=-1) + f0, ax0 = single_plot(wrms_results, names=names, ylabel='Weighted RMSE', bars=bars, percentile=25) + f1, ax1 = single_plot(true_pos_results, names=names, ylabel='True positive selections (out of 30)', bars=True, precut=1, percentile=25) + f2, ax2 = single_plot(selected_error, names=names, ylabel='RMSE of best paths', bars=True, precut=1, percentile=25) f = [f0, f1, f2] ax = [ax0, ax1, ax2] @@ -92,30 +96,41 @@ def _save_file(file, data): pickle.dump(data, fh) return - -def save_data(data_dir, wrms, true_pos, sel_err, obs, rel_err=None): - _save_file(data_dir + 'wrms.pkl', wrms) - _save_file(data_dir + 'true_pos.pkl', true_pos) - _save_file(data_dir + 'selected_error.pkl', sel_err) +def save_data(data_dir, wrms, true_pos, sel_err, obs, rel_err=None, full_obs=None, t=None): + if t is None: + t = wrms.shape[2] + _save_file(data_dir + 'wrms.pkl', wrms[:,:,:t]) + _save_file(data_dir + 'true_pos.pkl', true_pos[:,:,:t]) + _save_file(data_dir + 'selected_error.pkl', sel_err[:,:,:t]) _save_file(data_dir + 'obs.pkl', obs) if rel_err is not None: - _save_file(data_dir + 'relative_error.pkl', rel_err) - + _save_file(data_dir + 'relative_error.pkl', rel_err[:,:,:t]) + if full_obs is not None: + fobs = {key:full_obs[key][:t] for key in full_obs} + _save_file(data_dir + 'full_obs.pkl', fobs) def _load_file(file): with open(file, 'rb') as fh: data = pickle.load(fh) # Dimensions n_learners, n_queries+1, n_trials return data +def _load_params(file): + with open(file, 'rt') as fh: + params = yaml.safe_load(fh) + return params -def load_data(data_dir=None): - if data_dir is None: - Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing - data_dir = askdirectory(initialdir='./data/') +def get_data_dir(): + Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing + data_dir = askdirectory(initialdir='./data/') if data_dir == '': # Cancel clicked raise IOError('No directory selected') elif data_dir[-1] is not '/': data_dir += '/' + return data_dir + +def load_data(data_dir=None): + if data_dir is None: + data_dir = get_data_dir() wrms_results = _load_file(data_dir+'wrms.pkl') true_pos_results = _load_file(data_dir+'true_pos.pkl') @@ -126,13 +141,17 @@ def load_data(data_dir=None): except IOError: print "No relative error data found." relative_error = None - - return data_dir, wrms_results, true_pos_results, selected_error, obs_array, relative_error + try: + full_obs = _load_file(data_dir+'full_obs.pkl') + except IOError: + print "No full observation set found." + full_obs = None + return data_dir, wrms_results, true_pos_results, selected_error, obs_array, relative_error, full_obs def load_and_plot(save_plots=True, *args, **kwargs): try: - data_dir, wrms_results, true_pos_results, selected_error, obs_array, relative_error = load_data() + data_dir, wrms_results, true_pos_results, selected_error, obs_array, relative_error, full_obs = load_data() except IOError: return None @@ -143,7 +162,7 @@ def load_and_plot(save_plots=True, *args, **kwargs): return hf def get_selection(): - _dd, wrms, true_pos, sel_err, obs, rel_err = load_data() + _dd, wrms, true_pos, sel_err, obs, rel_err, full_obs = load_data() print "The following models were found:" for i, obs_item in enumerate(obs): print "{0}: {1}".format(i, obs_item['name']) @@ -174,5 +193,108 @@ def load_multiple(*args, **kwargs): print 'Data saved to {0}'.format(kwargs['data_dir']) return hf, wrms, true_pos, sel_err, obs, rel_err + +def build_ordinal_wrms(max_y = 0.8, *args, **kwargs): + data_dir = get_data_dir() + run_parameters = _load_params(data_dir+'params.yaml') + wave_data = _load_file(data_dir + 'wave_data.pkl') + full_obs = _load_file(data_dir + 'full_obs.pkl') + random_wave = test_data.MultiWave(**run_parameters['wave_params']) + + if full_obs is None: + raise IOError('full_obs.pkl not found, cannot reconstruct without full observation history') + + log_hyp = np.log(run_parameters['hyperparameters']) + n_learners, n_queries = len(run_parameters['learners']), run_parameters['statrun_params']['n_queries'] + n_trials = len(full_obs[run_parameters['learners'][0]['name']]) + x_plot = np.linspace(0.0, 1.0, run_parameters['statrun_params']['n_xtest'], dtype='float') + x_test = np.atleast_2d(x_plot).T + + wkld = np.zeros((n_learners, n_queries+1, n_trials), dtype='float') + max_count = np.zeros((n_learners, n_queries+1, n_trials), dtype='float') + + learners, names = [], [] + for l in run_parameters['learners']: + if 'n_rel_samples' in l['obs_args']: + l['obs_args']['n_rel_samples'] = run_parameters['statrun_params']['n_rel_samples'] + learners.append(active_learners.Learner(**l)) + names.append(l['name']) + + for trial_number in range(n_trials): + print 'Trial {0}'.format(trial_number) + a, f, o, d = wave_data.amplitude[trial_number], wave_data.frequency[trial_number], wave_data.offset[trial_number], wave_data.damping[trial_number] + random_wave.set_values(a, f, o, d) + random_wave.print_values() + abs_obs_fun = GPpref.AbsObservationSampler(random_wave.out, run_parameters['GP_params']['abs_likelihood'], + run_parameters['abs_obs_params']) + + min_max_label = np.floor(max_y * abs_obs_fun.l.y_list[-1]) + + p_y_true = abs_obs_fun.observation_likelihood_array(x_test) + true_y = abs_obs_fun.l.y_list[p_y_true.argmax(axis=0)] # True maximum likelihood labels + max_y_true = (true_y >= min_max_label) # True best label indexes (bool array) + n_max = float(max_y_true.sum()) + + for obs_num in range(0, n_queries+1): + for nl, learner in enumerate(learners): + # Get initial observations and build models + if obs_num == 0: + obs0 = full_obs[learner.name][trial_number][0] + x_rel, uvi_rel, x_abs, y_rel, y_abs = obs0.x_rel, obs0.uvi_rel, obs0.x_abs, obs0.y_rel, obs0.y_abs + model_kwargs = {'x_rel': x_rel, 'uvi_rel': uvi_rel, 'x_abs': x_abs, 'y_rel': y_rel, 'y_abs': y_abs, + 'rel_kwargs': run_parameters['rel_obs_params'], + 'abs_kwargs': run_parameters['abs_obs_params']} + model_kwargs.update(run_parameters['GP_params']) + learner.build_model(model_kwargs) + learner.model.set_hyperparameters(log_hyp) + + else: + next_x, next_y, next_uvi = full_obs[learner.name][trial_number][obs_num] + learner.model.add_observations(next_x, next_y, next_uvi) + + learner.model.solve_laplace() + fhat, vhat = learner.model.predict_latent(x_test) + p_y_est, mu_est = learner.model.abs_likelihood.posterior_likelihood(fhat, vhat) + est_y = abs_obs_fun.l.y_list[p_y_est.argmax(axis=0)] + max_y_est = (est_y >= min_max_label) + + wkld[nl, obs_num, trial_number] = test_data.ordinal_kld(p_y_true, p_y_est, np.maximum(true_y, est_y)) + max_count[nl, obs_num, trial_number] = np.logical_and(max_y_true, max_y_est).sum() / n_max + + + _save_file(data_dir+'wkld.pkl', wkld) + _save_file(data_dir+'max_count.pkl', max_count) + plot_ordinal_results(wkld, max_count, run_parameters=run_parameters, data_dir=data_dir) + +def plot_ordinal_results(wkld, max_count, run_parameters = None, data_dir=None, bars=True, exclusions=[]): + if run_parameters is None: + with open(data_dir + 'params.yaml', 'rt') as fh: + run_parameters = yaml.safe_load(fh) + + names = [] + for l in run_parameters['learners']: + names.append(l['name']) + + methods_indexes = [] + for i in range(len(names)): + if i not in exclusions: + methods_indexes.append(i) + methods_indexes = np.array(methods_indexes) + + wkld=wkld[methods_indexes,:,:] + max_count=max_count[methods_indexes,:,:] + + f0, ax0 = single_plot(wkld, names=names, ylabel='Weighted KLD', bars=bars, percentile=-1) + f1, ax1 = single_plot(max_count, names=names, ylabel='Proportion of selected max points', bars=True, precut=1, percentile=-1) + f = [f0, f1] + ax = [ax0, ax1] + + if data_dir is not None: + f0.savefig(data_dir + 'wkld.pdf', bbox_inches='tight') + f1.savefig(data_dir + 'max_count.pdf', bbox_inches='tight') + plt.show() + return f + if __name__ == "__main__": + # build_ordinal_wrms() hf = load_and_plot(save_plots=False, bars=True) \ No newline at end of file diff --git a/plot_tools.py b/plot_tools.py index 2726cf6..937ebfb 100644 --- a/plot_tools.py +++ b/plot_tools.py @@ -139,8 +139,8 @@ def estimate_plots(xt, ft, mu_t, fhat, vhat, E_y, rel_sigma, # Posterior samples if n_posterior_samples > 0: - y_post = np.random.multivariate_normal(fhat.flatten(), vhat, n_posterior_samples) - h_pp = ax_l.plot(xt, y_post.T, lw=0.8) + f_post = np.random.multivariate_normal(fhat.flatten(), vhat, n_posterior_samples) + h_pp = ax_l.plot(xt, f_post.T, lw=0.8) # Latent function hf, hpf = plot_with_bounds(ax_l, xt, ft, rel_sigma, c=lines[0]) diff --git a/pref_active_learning.py b/pref_active_learning.py index 02102da..52c259f 100644 --- a/pref_active_learning.py +++ b/pref_active_learning.py @@ -15,7 +15,7 @@ save_plots = False -with open('./data/ordinal_test.yaml', 'rt') as fh: +with open('./data/statruns_nov2017.yaml', 'rt') as fh: wave = yaml.safe_load(fh) try: @@ -30,6 +30,7 @@ n_xplot = 101 keep_f = True +learner_index = -1 log_hyp = np.log(wave['hyperparameters']) @@ -77,9 +78,10 @@ # Construct GP object prefGP = GPpref.PreferenceGaussianProcess(x_rel, uvi_rel, x_abs, y_rel, y_abs, **wave['GP_params']) -model_kwargs = {'x_rel':x_rel, 'uvi_rel':uvi_rel, 'x_abs':x_abs, 'y_rel':y_rel, 'y_abs':y_abs} +model_kwargs = {'x_rel':x_rel, 'uvi_rel':uvi_rel, 'x_abs':x_abs, 'y_rel':y_rel, 'y_abs':y_abs, + 'rel_kwargs': wave['rel_obs_params'], 'abs_kwargs': wave['abs_obs_params']} model_kwargs.update(wave['GP_params']) -learner_kwargs = wave['learners'][-1] +learner_kwargs = wave['learners'][learner_index] learner = active_learners.Learner(**learner_kwargs) learner.build_model(model_kwargs) diff --git a/test_data.py b/test_data.py index add42e3..dba183a 100644 --- a/test_data.py +++ b/test_data.py @@ -11,13 +11,12 @@ def wrms(y_true, y_est, weight=True): return np.sqrt(np.mean(((y_true - y_est)*w)**2)) -def wrms_misclass(y_true, y_est): +def wrms_misclass(y_true, y_est, w_power=2): # This is the misclassification error, where the weight is the max of the true or predicted value (penalise # predicting high values if the true value is low) - w = np.power(np.maximum(y_true, y_est), 2) + w = np.power(np.maximum(y_true, y_est), w_power) return np.sqrt(np.mean(((y_true - y_est)*w)**2)) - def rel_error(y_true, prel_true, y_est, prel_est, weight=False): if weight: y_max = np.maximum(y_true.flatten(), y_est.flatten()) @@ -33,11 +32,18 @@ def rel_error(y_true, prel_true, y_est, prel_est, weight=False): mean_p_err += w*np.abs(prel_true[i,j] - prel_est[i,j]) return mean_p_err/w_sum +def ordinal_kld(p_y_true, p_y_est, w = np.array([1.0])): + kld = -(p_y_true * np.log(p_y_est/p_y_true)).sum(axis=0) + wkld = w*kld/w.mean() + return wkld.mean() class ObsObject(object): def __init__(self, x_rel, uvi_rel, x_abs, y_rel, y_abs): self.x_rel, self.uvi_rel, self.x_abs, self.y_rel, self.y_abs = x_rel, uvi_rel, x_abs, y_rel, y_abs + def get_obs(self): + return self.x_rel, self.uvi_rel, self.x_abs, self.y_rel, self.y_abs + def obs_stats(obs_array, n_rel_samples): for method in obs_array: n_rel = 0.0 @@ -143,6 +149,9 @@ def save(self, filename): with open(filename, 'wb') as fh: pickle.dump(self, fh) + def get_vals(self, n): + return self.amplitude[n], self.frequency[n], self.offset[n], self.damping[n] + def damped_wave(x): y = np.cos(6 * np.pi * (x - 0.5)) * np.exp(-10 * (x - 0.5) ** 2) return y @@ -155,7 +164,6 @@ def basic_sine(x): y = (np.sin(x*2*np.pi + np.pi/4))/1.2 return y - def zero_fun(x): return 0*x From ac1abbb359a2f31a1d329d71a7bc7538e85c46ae Mon Sep 17 00:00:00 2001 From: Nicholas Lawrance Date: Fri, 2 Feb 2018 11:02:38 +0100 Subject: [PATCH 21/38] Fixed GP_preference_demo Updated to use obs_kwargs (primarily to know n_ordinals) during construction of PreferenceGP object --- GP_preference_demo.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/GP_preference_demo.py b/GP_preference_demo.py index 88345d6..bd058f4 100644 --- a/GP_preference_demo.py +++ b/GP_preference_demo.py @@ -56,7 +56,10 @@ # Construct GP object wave['GP_params']['verbose'] = verbose -prefGP = GPpref.PreferenceGaussianProcess(x_rel, uvi_rel, x_abs, y_rel, y_abs, **wave['GP_params']) +model_kwargs = {'x_rel':x_rel, 'uvi_rel':uvi_rel, 'x_abs':x_abs, 'y_rel':y_rel, 'y_abs':y_abs, + 'rel_kwargs': wave['rel_obs_params'], 'abs_kwargs': wave['abs_obs_params']} +model_kwargs.update(wave['GP_params']) +prefGP = GPpref.PreferenceGaussianProcess(**model_kwargs) prefGP.set_hyperparameters(log_hyp) # If training hyperparameters, use external optimiser From 56c9abce2720aa0ce63cf07a9e9b3f32508eb212 Mon Sep 17 00:00:00 2001 From: Nicholas Lawrance Date: Mon, 5 Feb 2018 16:42:34 +0100 Subject: [PATCH 22/38] Remote statrun fixes Added some stuff to run statrun tools remotely (command line options to disable plots), still working on higher dimensions of input, currently removing uv_rel stuff because it is pointless (just use x_rel and uvi) --- GP_preference_demo.py | 6 +++--- GPpref.py | 2 +- active_statruns.py | 16 ++++++++++++++-- plot_statruns.py | 40 +++++++++++++++++++++++++++++++++------- test_data.py | 15 ++++++++------- 5 files changed, 59 insertions(+), 20 deletions(-) diff --git a/GP_preference_demo.py b/GP_preference_demo.py index bd058f4..d862f82 100644 --- a/GP_preference_demo.py +++ b/GP_preference_demo.py @@ -14,14 +14,14 @@ use_test_data = False # test_data.data3 # verbose = 2 -with open('./data/statruns_dec2017.yaml', 'rt') as fh: +with open('./data/statruns_jan2018.yaml', 'rt') as fh: wave = yaml.safe_load(fh) - try: np.random.seed(wave['statrun_params']['randseed']) except KeyError: np.random.seed(0) -random_wave = test_data.MultiWave(**wave['wave_params']) +d_x = wave['GP_params']['hyper_counts'][0]-1 +random_wave = test_data.MultiWave(n_dimensions=d_x, **wave['wave_params']) log_hyp = np.log(wave['hyperparameters']) n_rel_train = 10 diff --git a/GPpref.py b/GPpref.py index 1c4bd3b..2e00868 100644 --- a/GPpref.py +++ b/GPpref.py @@ -719,7 +719,7 @@ def observation_likelihood_array(self, x, y=-1): def generate_n_observations(self, n, n_xdim=1, domain=None): x = self._gen_x_obs(2*n, n_xdim, domain) uvi = np.arange(2*n).reshape((n, 2)) - uv = x[uvi][:, :, 0] + uv = x[uvi] # [:, :, 0] y, fuv = self.generate_observations(uv) return x, uvi, uv, y, fuv diff --git a/active_statruns.py b/active_statruns.py index f3e52dc..2d85b1e 100644 --- a/active_statruns.py +++ b/active_statruns.py @@ -7,14 +7,25 @@ import test_data import plot_statruns import yaml +import argparse np.set_printoptions(precision=3) wrms_fun = test_data.wrms_misclass wrms_args = {'w_power': 1} now_time = time.strftime("%Y_%m_%d-%H_%M") +yaml_config ='./data/statruns_dec2017.yaml' -with open('./data/statruns_nov2017.yaml', 'rt') as fh: +parser = argparse.ArgumentParser(description='Statruns for active learning with preference GP') +parser.add_argument('-np', '--no-plots', dest='make_plots', action='store_false', help='Turn off plots (default False)') +parser.add_argument('yaml_config', default=yaml_config, help='YAML config file') + +args = parser.parse_args() + +print "Using YAML config: {0}".format(args.yaml_config) +if not args.make_plots: + print "No plot output." +with open(args.yaml_config, 'rt') as fh: run_parameters = yaml.safe_load(fh) log_hyp = np.log(run_parameters['hyperparameters']) @@ -175,4 +186,5 @@ waver.save(data_dir+'wave_data.pkl') -hfig = plot_statruns.plot_results(wrms_results, true_pos_results, selected_error, obs_array, relative_error=relative_error, data_dir=data_dir, bars=True, norm_comparator=0) \ No newline at end of file +if args.make_plots: + hfig = plot_statruns.plot_results(wrms_results, true_pos_results, selected_error, obs_array, relative_error=relative_error, data_dir=data_dir, bars=True, norm_comparator=0) \ No newline at end of file diff --git a/plot_statruns.py b/plot_statruns.py index 5480e86..eefa2ef 100644 --- a/plot_statruns.py +++ b/plot_statruns.py @@ -7,6 +7,7 @@ import active_learners from Tkinter import Tk from tkFileDialog import askdirectory +import argparse # from test_data import ObsObject plt.rc('font',**{'family':'serif','sans-serif':['Computer Modern Roman']}) plt.rc('text', usetex=True) @@ -149,9 +150,9 @@ def load_data(data_dir=None): return data_dir, wrms_results, true_pos_results, selected_error, obs_array, relative_error, full_obs -def load_and_plot(save_plots=True, *args, **kwargs): +def load_and_plot(save_plots=True, data_dir=None, *args, **kwargs): try: - data_dir, wrms_results, true_pos_results, selected_error, obs_array, relative_error, full_obs = load_data() + data_dir, wrms_results, true_pos_results, selected_error, obs_array, relative_error, full_obs = load_data(data_dir) except IOError: return None @@ -194,8 +195,9 @@ def load_multiple(*args, **kwargs): return hf, wrms, true_pos, sel_err, obs, rel_err -def build_ordinal_wrms(max_y = 0.8, *args, **kwargs): - data_dir = get_data_dir() +def build_ordinal_wrms(max_y = 0.8, data_dir = None, make_plots = True, *args, **kwargs): + if data_dir is None: + data_dir = get_data_dir() run_parameters = _load_params(data_dir+'params.yaml') wave_data = _load_file(data_dir + 'wave_data.pkl') full_obs = _load_file(data_dir + 'full_obs.pkl') @@ -264,7 +266,17 @@ def build_ordinal_wrms(max_y = 0.8, *args, **kwargs): _save_file(data_dir+'wkld.pkl', wkld) _save_file(data_dir+'max_count.pkl', max_count) - plot_ordinal_results(wkld, max_count, run_parameters=run_parameters, data_dir=data_dir) + if make_plots: + plot_ordinal_results(wkld, max_count, run_parameters=run_parameters, data_dir=data_dir) + +def load_ordinal_results(data_dir = None): + if data_dir is None: + data_dir = get_data_dir() + + wkld = _load_file(data_dir+'wkld.pkl') + max_count = _load_file(data_dir+'max_count.pkl') + + plot_ordinal_results(wkld, max_count, data_dir=data_dir) def plot_ordinal_results(wkld, max_count, run_parameters = None, data_dir=None, bars=True, exclusions=[]): if run_parameters is None: @@ -296,5 +308,19 @@ def plot_ordinal_results(wkld, max_count, run_parameters = None, data_dir=None, return f if __name__ == "__main__": - # build_ordinal_wrms() - hf = load_and_plot(save_plots=False, bars=True) \ No newline at end of file + parser = argparse.ArgumentParser(description='Statrun plots for active learning preference GP') + parser.add_argument('-np', '--no-plots', dest='make_plots', action='store_false', + help='Turn off plots (default False)') + parser.add_argument('-ow', '--ordinal-wrms', dest='owrms', action='store_true', + help='Build ordinal wrms results (default False)') + parser.add_argument('-lm', '--load-multiple', dest='load_multiple', action='store_true', + help='Load from multiple directories (default False)') + parser.add_argument('-d', '--data-dir', default=None, help='Data directory') + args = parser.parse_args() + + if args.owrms: + build_ordinal_wrms(data_dir=args.data_dir, make_plots=args.make_plots) + if args.load_multiple: + load_multiple(save_plots=args.make_plots) + else: + load_and_plot(save_plots=args.make_plots, bars=True) \ No newline at end of file diff --git a/test_data.py b/test_data.py index dba183a..e46a29b 100644 --- a/test_data.py +++ b/test_data.py @@ -55,12 +55,13 @@ def obs_stats(obs_array, n_rel_samples): class VariableWave(object): - def __init__(self, amp_range, f_range, off_range, damp_range, n_components=1): + def __init__(self, amp_range, f_range, off_range, damp_range, n_components=1, n_dimensions=1): self.amp_range = amp_range self.f_range = f_range self.off_range = off_range self.damp_range = damp_range self.n_components = n_components + self.n_dimensions = n_dimensions self.randomize() def out(self, x): @@ -93,13 +94,13 @@ def out(self, x): y = np.zeros(np.array(x).shape) for a, f, o, d in zip(self.amplitude, self.frequency, self.offset, self.damping): y += a*np.cos(f*np.pi*(x-o)) * np.exp(-d*(x-o)**2) - return y + return y.sum(axis=1, keepdims=True) def randomize(self, print_vals=False): - self.amplitude = np.random.uniform(low=self.amp_range[0], high=self.amp_range[1], size=self.n_components) - self.frequency = np.random.uniform(low=self.f_range[0], high=self.f_range[1], size=self.n_components) - self.offset = np.random.uniform(low=self.off_range[0], high=self.off_range[1], size=self.n_components) - self.damping = np.random.uniform(low=self.damp_range[0], high=self.damp_range[1], size=self.n_components) + self.amplitude = np.random.uniform(low=self.amp_range[0], high=self.amp_range[1], size=(self.n_components, self.n_dimensions)) + self.frequency = np.random.uniform(low=self.f_range[0], high=self.f_range[1], size=(self.n_components, self.n_dimensions)) + self.offset = np.random.uniform(low=self.off_range[0], high=self.off_range[1], size=(self.n_components, self.n_dimensions)) + self.damping = np.random.uniform(low=self.damp_range[0], high=self.damp_range[1], size=(self.n_components, self.n_dimensions)) if print_vals: self.print_values() @@ -203,7 +204,7 @@ def data3(): [0.06054717], [0.45331369], [0.8461625 ], [0.58854979]]) uvi_rel = np.array([[0, 1], [2, 3], [4, 5]], dtype='int') - uv_rel = x_rel[uvi_rel][:,:,0] + uv_rel = x_rel[uvi_rel] # [:,:,0] y_rel = np.array([[-1], [1], [1]], dtype='int') fuv_rel = np.array([[0.0043639, -0.10653237], [0.01463141, 0.05046293], [0.01773679, 0.45730181]]) From a1749175578fc84ce2e53fa2000cb6bc00a2b3a9 Mon Sep 17 00:00:00 2001 From: Nicholas Lawrance Date: Wed, 7 Feb 2018 17:56:53 +0100 Subject: [PATCH 23/38] Added methods for 2D plots, still pretty hard to interpret but working --- GP_preference_demo.py | 77 ++++++++++++------- GPpref.py | 11 ++- plot_tools.py | 170 +++++++++++++++++++++++++++--------------- test_data.py | 12 +-- 4 files changed, 173 insertions(+), 97 deletions(-) diff --git a/GP_preference_demo.py b/GP_preference_demo.py index d862f82..20eeaa4 100644 --- a/GP_preference_demo.py +++ b/GP_preference_demo.py @@ -14,7 +14,7 @@ use_test_data = False # test_data.data3 # verbose = 2 -with open('./data/statruns_jan2018.yaml', 'rt') as fh: +with open('./data/statruns_2D.yaml', 'rt') as fh: wave = yaml.safe_load(fh) try: np.random.seed(wave['statrun_params']['randseed']) @@ -24,10 +24,10 @@ random_wave = test_data.MultiWave(n_dimensions=d_x, **wave['wave_params']) log_hyp = np.log(wave['hyperparameters']) -n_rel_train = 10 -n_abs_train = 20 +n_rel_train = 50 +n_abs_train = 5 -n_xplot = 101 +n_xplot = 21 n_posterior_samples = 3 random_wave.print_values() @@ -37,22 +37,25 @@ abs_obs_fun = GPpref.AbsObservationSampler(true_function, wave['GP_params']['abs_likelihood'], wave['abs_obs_params']) # True function +# This is a complicated (but memory efficient, maybe?) way to generate list of all grid points, there's probably a +# better (itertools way) x_plot = np.linspace(0.0,1.0,n_xplot,dtype='float') -x_test = np.atleast_2d(x_plot).T +x_test = ptt.make_meshlist(x_plot, d_x) f_true = abs_obs_fun.f(x_test) mu_true = abs_obs_fun.mean_link(x_test) abs_y_samples = abs_obs_fun.l.y_list p_abs_y_true = abs_obs_fun.observation_likelihood_array(x_test, abs_y_samples) -p_rel_y_true = rel_obs_fun.observation_likelihood_array(x_test) +if d_x is 1: + p_rel_y_true = rel_obs_fun.observation_likelihood_array(x_test) # Training data - this is a bit weird, but we sample x values, then the uv pairs # are actually indexes into x, because it is easier computationally. You can # recover the actual u,v values using x[ui],x[vi] if use_test_data: - x_rel, uvi_rel, uv_rel, y_rel, fuv_rel, x_abs, y_abs, mu_abs = use_test_data() + x_rel, uvi_rel, y_rel, fuv_rel, x_abs, y_abs, mu_abs = use_test_data() else: - x_rel, uvi_rel, uv_rel, y_rel, fuv_rel = rel_obs_fun.generate_n_observations(n_rel_train) - x_abs, y_abs, mu_abs = abs_obs_fun.generate_n_observations(n_abs_train) + x_rel, uvi_rel, y_rel, fuv_rel = rel_obs_fun.generate_n_observations(n_rel_train, n_xdim=d_x) + x_abs, y_abs, mu_abs = abs_obs_fun.generate_n_observations(n_abs_train, n_xdim=d_x) # Construct GP object wave['GP_params']['verbose'] = verbose @@ -74,28 +77,48 @@ # Posterior likelihoods p_abs_y_post, E_y = prefGP.abs_posterior_likelihood(abs_y_samples, fhat=fhat, varhat=vhat) -p_rel_y_post = prefGP.rel_posterior_likelihood_array(fhat=fhat, varhat=vhat) - - -# Plot true functions -fig_t, (ax_t_l, ax_t_a, ax_t_r) = ptt.true_plots(x_test, f_true, mu_true, wave['rel_obs_params']['sigma'], - abs_y_samples, p_abs_y_true, p_rel_y_true, - t_l=r'True latent function, $f(x)$') - -# Posterior estimates -fig_p, (ax_p_l, ax_p_a, ax_p_r) = \ - ptt.estimate_plots(x_test, f_true, mu_true, fhat, vhat, E_y, wave['rel_obs_params']['sigma'], - abs_y_samples, p_abs_y_post, p_rel_y_post, - x_abs, y_abs, uv_rel, fuv_rel, y_rel, n_posterior_samples=n_posterior_samples, - t_a=r'Posterior absolute likelihood, $p(u | \mathcal{Y}, \theta)$', - t_r=r'Posterior relative likelihood $P(x_0 \succ x_1 | \mathcal{Y}, \theta)$') wrms = test_data.wrms(mu_true, E_y) wrms2 = test_data.wrms_misclass(mu_true, E_y) -p_err = test_data.rel_error(mu_true, p_rel_y_true, E_y, p_rel_y_post, weight=False) -print "WRMS: {0:0.3f}, WRMS_MC: {1:0.3f}, p_err: {2:0.3f}".format(wrms, wrms2, p_err) -plt.show() +if d_x is 1: + p_rel_y_post = prefGP.rel_posterior_likelihood_array(fhat=fhat, varhat=vhat) + + # Plot true functions + fig_t, (ax_t_l, ax_t_a, ax_t_r) = ptt.true_plots(x_test, f_true, mu_true, wave['rel_obs_params']['sigma'], + abs_y_samples, p_abs_y_true, p_rel_y_true, + t_l=r'True latent function, $f(x)$') + + # Posterior estimates + fig_p, (ax_p_l, ax_p_a, ax_p_r) = \ + ptt.estimate_plots(x_test, f_true, mu_true, fhat, vhat, E_y, wave['rel_obs_params']['sigma'], + abs_y_samples, p_abs_y_post, p_rel_y_post, + x_abs, y_abs, x_rel[uvi_rel][:,:,0], fuv_rel, y_rel, n_posterior_samples=n_posterior_samples, + t_a=r'Posterior absolute likelihood, $p(u | \mathcal{Y}, \theta)$', + t_r=r'Posterior relative likelihood $P(x_0 \succ x_1 | \mathcal{Y}, \theta)$') + p_err = test_data.rel_error(mu_true, p_rel_y_true, E_y, p_rel_y_post, weight=False) + print "WRMS: {0:0.3f}, WRMS_MC: {1:0.3f}, p_err: {2:0.3f}".format(wrms, wrms2, p_err) + plt.show() + + +elif d_x is 2: + # Plot true functions + fig_t, (ax_t_l, ax_t_a) = ptt.true_plots2D(x_test, f_true, mu_true, wave['rel_obs_params']['sigma'], + abs_y_samples, p_abs_y_true, t_l=r'True latent function, $f(x)$') + + # Posterior estimates + fig_p, (ax_p_l, ax_p_a) = \ + ptt.estimate_plots2D(x_test, f_true, mu_true, fhat, vhat, E_y, wave['rel_obs_params']['sigma'], + abs_y_samples, p_abs_y_post, x_abs, y_abs, x_rel[uvi_rel], fuv_rel, y_rel, + t_l=r'$\mathcal{GP}$ latent function estimate $\hat{f}(x)$', + t_a=r'Posterior absolute likelihood, $p(u | \mathcal{Y}, \theta)$') + print "WRMS: {0:0.3f}, WRMS_MC: {1:0.3f}".format(wrms, wrms2) + plt.show() + + +else: + print "Input state space dimension: {0} is too high to plot".format(d_x) + print "WRMS: {0:0.3f}, WRMS_MC: {1:0.3f}".format(wrms, wrms2) ## SCRAP diff --git a/GPpref.py b/GPpref.py index 2e00868..6664c29 100644 --- a/GPpref.py +++ b/GPpref.py @@ -716,12 +716,17 @@ def observation_likelihood_array(self, x, y=-1): p_y[:, i:i + 1] = self.l.likelihood(y, cross_fx) return p_y + def generate_observations(self, x, uvi): + fx = self.f(x) + y, ff = self.l.generate_samples(fx[uvi][:, :, 0]) + return y, ff + def generate_n_observations(self, n, n_xdim=1, domain=None): x = self._gen_x_obs(2*n, n_xdim, domain) uvi = np.arange(2*n).reshape((n, 2)) - uv = x[uvi] # [:, :, 0] - y, fuv = self.generate_observations(uv) - return x, uvi, uv, y, fuv + # uv = x[uvi] # [:, :, 0] + y, fuv = self.generate_observations(x, uvi) + return x, uvi, y, fuv diff --git a/plot_tools.py b/plot_tools.py index 937ebfb..4336023 100644 --- a/plot_tools.py +++ b/plot_tools.py @@ -2,12 +2,23 @@ import numpy as np import matplotlib.pyplot as plt from matplotlib.patches import Polygon +import matplotlib.cm as cm from nice_plot_colors import * from cycler import cycler +from mpl_toolkits.mplot3d.art3d import Line3DCollection # plt.rc('axes', prop_cycle=(cycler('color', [greyify(c, .5, .8) for c in reversed(lines)]))) plt.rc('axes', prop_cycle=(cycler('color', lines))) +def xgen(xp, n_dim): + for i in range(n_dim): + yield xp + +def make_meshlist(x_plot, d_x): + xx = xgen(x_plot, d_x) + x_mesh = np.vstack(np.meshgrid(*xx, copy=False)).reshape(d_x, -1).T + return x_mesh + def make_poly_array(x,y,sigma): nx = len(x) sigma = np.atleast_2d(sigma) @@ -32,23 +43,7 @@ def plot_with_bounds(ax, x, y, s, c=lines[0]): ax.set_ylim(bottom = min(clim[0],xy[:,1].min()), top = max(clim[1], xy[:,1].max())) return h_fx, h_patch -def plot_setup_rel(t_l = r'Latent function, $f(x)$', t_r = r'Relative likelihood, $P(x_0 \succ x_1 | f(x_0), f(x_1))$'): - - fig, (ax_l, ax_r) = plt.subplots(1, 2) - fig.set_size_inches(9.8, 3.5) - - # Latent function - ax_l.set_title(t_l) - ax_l.set_xlabel('$x$') - ax_l.set_ylabel('$f(x)$') - - # Relative likelihood - ax_r.set_title(t_r) - ax_r.set_xlabel('$x_0$') - ax_r.set_ylabel('$x_1$') - return fig, (ax_l, ax_r) - -def plot_setup_2d(t_l = r'Latent function, $f(x)$', t_a = r'Absolute likelihood, $p(y | f(x))$', +def plot_setup_1d(t_l = r'Latent function, $f(x)$', t_a = r'Absolute likelihood, $p(y | f(x))$', t_r = r'Relative likelihood, $P(x_0 \succ x_1 | f(x_0), f(x_1))$'): fig, (ax_l, ax_a, ax_r) = plt.subplots(1, 3) @@ -70,7 +65,6 @@ def plot_setup_2d(t_l = r'Latent function, $f(x)$', t_a = r'Absolute likelihood, ax_r.set_ylabel('$x_1$') return fig, (ax_l, ax_a, ax_r) - def plot_relative_likelihood(ax, p_y, extent): h_p = ax.imshow(p_y, origin='lower', extent=extent, vmin=0.0, vmax=1.0) h_pc = ax.contour(p_y, levels=[0.5], origin='lower', linewidths=2, extent=extent) @@ -83,7 +77,7 @@ def true_plots(xt, ft, mu_t, rel_sigma, y_samples, p_a_y, p_r_y, xa_train=None, # Plot true function, likelihoods and observations - fig, (ax_l, ax_a, ax_r) = plot_setup_2d(**kwargs) + fig, (ax_l, ax_a, ax_r) = plot_setup_1d(**kwargs) # True latent plot_with_bounds(ax_l, xt, ft, rel_sigma, c=lines[0]) @@ -91,7 +85,8 @@ def true_plots(xt, ft, mu_t, rel_sigma, y_samples, p_a_y, p_r_y, xa_train=None, # True absolute likelihood dely = y_samples[1, 0] - y_samples[0, 0] abs_extent = [xt[0, 0], xt[-1, 0], y_samples[0, 0] - 0.5*dely, y_samples[-1, 0] + 0.5*dely] - h_pat = ax_a.imshow(p_a_y, origin='lower', extent=abs_extent, aspect='auto') + vmax = max(1.0, p_a_y.max()) + h_pat = ax_a.imshow(p_a_y, origin='lower', extent=abs_extent, aspect='auto', vmin=0.0, vmax=vmax) if xa_train is not None and xa_train.shape[0] > 0: ax_a.plot(xa_train, ya_train, 'w+') h_yt, = ax_a.plot(xt, mu_t, c=lines[0]) @@ -109,33 +104,10 @@ def true_plots(xt, ft, mu_t, rel_sigma, y_samples, p_a_y, p_r_y, xa_train=None, ax_l.plot(uv[(y + 1) / 2], fuv[(y + 1) / 2], class_icons[(y[0] + 1) / 2], **marker_options) return fig, (ax_l, ax_a, ax_r) - -def true_plots_rel(xt, ft, mu_t, rel_sigma, y_samples, p_a_y, p_r_y, xa_train, ya_train, uvr_train, fuvr_train, yr_train, - class_icons=['ko', 'wo'], marker_options={'mec':'k', 'mew':0.5}, *args, **kwargs): - - # Plot true function, likelihoods and observations - fig, (ax_l, ax_r) = plot_setup_rel(**kwargs) - - # True latent - plot_with_bounds(ax_l, xt, ft, rel_sigma, c=lines[0]) - - # True absolute likelihood - - # True relative likelihood - rel_y_extent = [xt[0, 0], xt[-1, 0], xt[0, 0], xt[-1, 0]] - h_prt = plot_relative_likelihood(ax_r, p_r_y, extent=rel_y_extent) - if xt.shape[0] > 0: - for uv, fuv, y in zip(uvr_train, fuvr_train, yr_train): - ax_r.plot(uv[0], uv[1], class_icons[(y[0] + 1) / 2], **marker_options) - ax_l.plot(uv, fuv, 'b-', color=lighten(lines[0])) - ax_l.plot(uv[(y + 1) / 2], fuv[(y + 1) / 2], class_icons[(y[0] + 1) / 2], **marker_options) - return fig, (ax_l, ax_r) - - def estimate_plots(xt, ft, mu_t, fhat, vhat, E_y, rel_sigma, y_samples, p_a_y, p_r_y, xa_train, ya_train, uvr_train, fuvr_train, yr_train, class_icons = ['ko', 'wo'], marker_options = {'mec':'k', 'mew':0.5}, n_posterior_samples=0, *args, **kwargs): - fig, (ax_l, ax_a, ax_r) = plot_setup_2d(**kwargs) + fig, (ax_l, ax_a, ax_r) = plot_setup_1d(**kwargs) # Posterior samples if n_posterior_samples > 0: @@ -152,6 +124,7 @@ def estimate_plots(xt, ft, mu_t, fhat, vhat, E_y, rel_sigma, # Absolute posterior likelihood dely = y_samples[1, 0]-y_samples[0, 0] abs_extent = [xt[0, 0], xt[-1, 0], y_samples[0, 0]-0.5*dely, y_samples[-1, 0]+0.5*dely] + vmax = max(1.0, p_a_y.max()) h_pap = ax_a.imshow(p_a_y, origin='lower', extent=abs_extent, aspect='auto') h_yt, = ax_a.plot(xt, mu_t, c=lines[0]) hEy, = ax_a.plot(xt, E_y, color=lines[3]) @@ -173,30 +146,105 @@ def estimate_plots(xt, ft, mu_t, fhat, vhat, E_y, rel_sigma, return fig, (ax_l, ax_a, ax_r) -def estimate_plots_rel(xt, ft, mu_t, fhat, vhat, E_y, rel_sigma, - y_samples, p_a_y, p_r_y, xa_train, ya_train, uvr_train, fuvr_train, yr_train, - class_icons = ['ko', 'wo'], marker_options = {'mec':'k', 'mew':0.5}, *args, **kwargs): - fig, (ax_l, ax_r) = plot_setup_rel(**kwargs) +# 2D Plot tools + +def plot_setup_2d(t_l = r'Latent function, $f(x)$', t_a = r'Absolute likelihood, $p(y | f(x))$'): + + fig = plt.figure() + ax_l = fig.add_subplot(121, projection='3d') + ax_a = fig.add_subplot(122, projection='3d') + fig.set_size_inches(10.0, 3.5) # Latent function - hf, hpf = plot_with_bounds(ax_l, xt, ft, rel_sigma, c=lines[0]) + ax_l.set_title(t_l) + ax_l.set_xlabel('$x_0$') + ax_l.set_ylabel('$x_1$') + ax_l.set_zlabel('$f(x)$') - hf_hat, hpf_hat = plot_with_bounds(ax_l, xt, fhat, np.sqrt(np.atleast_2d(vhat.diagonal()).T), c=lines[1]) - ax_l.legend([hf, hf_hat], [r'True latent function, $f(x)$', r'$\mathcal{GP}$ estimate $\hat{f}(x)$']) + # Absolute likelihood + ax_a.set_title(t_a) + ax_a.set_xlabel('$x_0$') + ax_a.set_ylabel('$x_1$') + ax_a.set_zlabel('$y$') + return fig, (ax_l, ax_a) - # Absolute posterior likelihood - abs_extent = [xt[0, 0], xt[-1, 0], y_samples[0, 0], y_samples[-1, 0]] +def true_plots2D(xt, ft, mu_t, rel_sigma, y_samples, p_a_y, xa_train=None, ya_train=None, uvr_train=None, fuvr_train=None, yr_train=None, + class_icons=['ko', 'wo'], marker_options={'mec':'k', 'mew':0.5}, *args, **kwargs): + + # Plot true function, likelihoods and observations + nx = int(np.sqrt(xt.shape[0])) + x0 = xt[0:nx, 0] # Assuming generated using make_meshlist + x1 = xt[0:-1:nx, 1] + xx, yy = np.meshgrid(x0, x1) + + fig, (ax_l, ax_a) = plot_setup_2d(**kwargs) + + # True latent + ax_l.plot_surface(xx, yy, np.reshape(ft, (nx, nx)), color=lines[0]) + # ax_l.imshow(np.reshape(ft, (nx, nx)), extent=[x0[0], x1[-1], x1[0], x1[1]], origin='lower', aspect='auto') + + # True absolute likelihood + h_yt = ax_a.plot_wireframe(xx, yy, np.reshape(mu_t, (nx, nx)), color=lines[1]) + + norm_py = p_a_y/p_a_y.max() + cc = cm.get_cmap() + for y, py in zip(y_samples, norm_py): + ax_a.scatter(xt[:, 0], xt[:, 1], y, s=py*15.0, marker='o', c=cc(py)) + + if xa_train is not None and xa_train.shape[0] > 0: + ax_a.scatter(xa_train[:, 0], xa_train[:, 1], ya_train, c='w', marker='+') + ax_a.legend([h_yt], [r'True mean $E[y]$']) + # ax_a.set_xlim(xt[0], xt[-1]) + + return fig, (ax_l, ax_a) + + +def estimate_plots2D(xt, ft, mu_t, fhat, vhat, E_y, rel_sigma, + y_samples, p_a_y, xa_train, ya_train, uvr_train, fuvr_train, yr_train, + class_icons = ['ko', 'wo'], marker_options = {'mec':'k', 'mew':0.5}, *args, **kwargs): + + # Plot estimated function, likelihoods and observations + nx = int(np.sqrt(xt.shape[0])) + x0 = xt[0:nx, 0] # Assuming generated using make_meshlist + x1 = xt[0:-1:nx, 1] + xx, yy = np.meshgrid(x0, x1) + + fig, (ax_l, ax_a) = plot_setup_2d(**kwargs) + cc = cm.get_cmap() + + # Latent function estimate + # hf =ax_l.plot_wireframe(xx, yy, np.reshape(ft, (nx, nx)), color=lines[0]) + tfc = np.reshape(vhat.diagonal(), (nx, nx)) + hf_hat = ax_l.plot_surface(xx, yy, np.reshape(fhat, (nx, nx)), facecolors=cc(tfc/tfc.max())) + # ax_l.imshow(np.reshape(fhat, (nx, nx)), extent=[x0[0], x1[-1], x1[0], x1[1]], origin='lower', aspect='auto') + # ax_l.legend([hf], [r'True latent function, $f(x)$']) #, r'$\mathcal{GP}$ estimate $\hat{f}(x)$']) - # Relative posterior likelihood - rel_y_extent = [xt[0, 0], xt[-1, 0], xt[0, 0], xt[-1, 0]] - h_prp = plot_relative_likelihood(ax_r, p_r_y, extent=rel_y_extent) if uvr_train.shape[0] > 0: - for uv, fuv, y in zip(uvr_train, fuvr_train, yr_train): - ax_r.plot(uv[0], uv[1], class_icons[(y[0] + 1) / 2], **marker_options) - ax_l.plot(uv, fuv, 'b-', color=lighten(lines[0])) - ax_l.plot(uv[(y + 1) / 2], fuv[(y + 1) / 2], class_icons[(y[0] + 1) / 2], **marker_options) + rel_segments = np.zeros((uvr_train.shape[0], 2, 3)) + rel_segments[:, :, 0:2] = uvr_train + rel_segments[:, :, 2] = fuvr_train + rel_lines = Line3DCollection(rel_segments, color=lines[3]) + ax_l.add_collection(rel_lines) + + rel_hipoints = np.array([uv[(i+1)/2] for uv, i in zip(rel_segments, yr_train.flat)]) + ax_l.plot(rel_hipoints[:, 0], rel_hipoints[:, 1], rel_hipoints[:, 2], 'ko', **marker_options) + + # Absolute posterior likelihood + # h_yt = ax_a.plot_wireframe(xx, yy, np.reshape(mu_t, (nx, nx)), color=lines[1]) + hEy = ax_a.plot_wireframe(xx, yy, np.reshape(E_y, (nx, nx)), color=lines[3]) + + norm_py = p_a_y/p_a_y.max() + for y, py in zip(y_samples, norm_py): + ax_a.scatter(xt[:, 0], xt[:, 1], y, s=py*15.0, marker='o', c=cc(py)) + + if xa_train.shape[0] > 0: + ax_a.plot(xa_train[:,0], xa_train[:,1], ya_train.flat, 'r^', color=lines[1]) + ax_a.legend([hEy], [r'Posterior mean, $E_{p(y|\mathcal{Y})}\left[y\right]$']) + # fig.colorbar(h_pap, ax=ax_a) + + + return fig, (ax_l, ax_a) - return fig, (ax_l, ax_r) def ensure_dir(file_path): directory = os.path.dirname(file_path) diff --git a/test_data.py b/test_data.py index e46a29b..1434cce 100644 --- a/test_data.py +++ b/test_data.py @@ -171,7 +171,7 @@ def zero_fun(x): def data1(): x_rel = np.array([[0.6], [0.7]]) uvi_rel = np.array([[0, 1], [1, 0]], dtype='int') - uv_rel = x_rel[uvi_rel][:,:,0] + # uv_rel = x_rel[uvi_rel][:,:,0] y_rel = np.array([[1], [1]], dtype='int') fuv_rel = np.array([[-0.1, 0.1], [-0.1, 0.1]]) @@ -179,7 +179,7 @@ def data1(): y_abs = np.array([[0.5]]) mu_abs = np.array([[0.0]]) - return x_rel, uvi_rel, uv_rel, y_rel, fuv_rel, x_abs, y_abs, mu_abs + return x_rel, uvi_rel, y_rel, fuv_rel, x_abs, y_abs, mu_abs def data2(): @@ -187,7 +187,7 @@ def data2(): [ 0.06054717], [ 0.45331369], [ 0.8461625 ], [ 0.58854979]]) uvi_rel = np.array([[0, 1], [2, 3], [4, 5]], dtype='int') - uv_rel = x_rel[uvi_rel][:,:,0] + # uv_rel = x_rel[uvi_rel][:,:,0] y_rel = np.array([[-1], [1], [1]], dtype='int') fuv_rel = np.array([[0.0043639, -0.10653237], [0.01463141, 0.05046293], [0.01773679, 0.45730181]]) @@ -196,7 +196,7 @@ def data2(): y_abs = np.array([[0.38966307]]) mu_abs = np.array([[0.0]]) - return x_rel, uvi_rel, uv_rel, y_rel, fuv_rel, x_abs, y_abs, mu_abs + return x_rel, uvi_rel, y_rel, fuv_rel, x_abs, y_abs, mu_abs def data3(): @@ -204,7 +204,7 @@ def data3(): [0.06054717], [0.45331369], [0.8461625 ], [0.58854979]]) uvi_rel = np.array([[0, 1], [2, 3], [4, 5]], dtype='int') - uv_rel = x_rel[uvi_rel] # [:,:,0] + # uv_rel = x_rel[uvi_rel][:,:,0] y_rel = np.array([[-1], [1], [1]], dtype='int') fuv_rel = np.array([[0.0043639, -0.10653237], [0.01463141, 0.05046293], [0.01773679, 0.45730181]]) @@ -213,4 +213,4 @@ def data3(): y_abs = np.array([[0.38966307], [0.999]]) mu_abs = np.array([[0.0], [0.0]]) - return x_rel, uvi_rel, uv_rel, y_rel, fuv_rel, x_abs, y_abs, mu_abs + return x_rel, uvi_rel, y_rel, fuv_rel, x_abs, y_abs, mu_abs From eee3fcd7bd6051bd711b12b72014dc0fdc068efe Mon Sep 17 00:00:00 2001 From: Nicholas Lawrance Date: Fri, 23 Mar 2018 14:04:32 +0100 Subject: [PATCH 24/38] Some 2D changes --- GP_preference_demo.py | 4 ++-- plot_tools.py | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/GP_preference_demo.py b/GP_preference_demo.py index 20eeaa4..79ba404 100644 --- a/GP_preference_demo.py +++ b/GP_preference_demo.py @@ -24,8 +24,8 @@ random_wave = test_data.MultiWave(n_dimensions=d_x, **wave['wave_params']) log_hyp = np.log(wave['hyperparameters']) -n_rel_train = 50 -n_abs_train = 5 +n_rel_train = 15 +n_abs_train = 2 n_xplot = 21 n_posterior_samples = 3 diff --git a/plot_tools.py b/plot_tools.py index 4336023..8fbd5e1 100644 --- a/plot_tools.py +++ b/plot_tools.py @@ -187,9 +187,10 @@ def true_plots2D(xt, ft, mu_t, rel_sigma, y_samples, p_a_y, xa_train=None, ya_tr h_yt = ax_a.plot_wireframe(xx, yy, np.reshape(mu_t, (nx, nx)), color=lines[1]) norm_py = p_a_y/p_a_y.max() - cc = cm.get_cmap() + cc = cm.get_cmap('Blues') + h_py = [] for y, py in zip(y_samples, norm_py): - ax_a.scatter(xt[:, 0], xt[:, 1], y, s=py*15.0, marker='o', c=cc(py)) + h_py.append(ax_a.scatter(xt[:, 0], xt[:, 1], y, s=py*15.0, marker='o', c=cc(py))) if xa_train is not None and xa_train.shape[0] > 0: ax_a.scatter(xa_train[:, 0], xa_train[:, 1], ya_train, c='w', marker='+') @@ -210,7 +211,7 @@ def estimate_plots2D(xt, ft, mu_t, fhat, vhat, E_y, rel_sigma, xx, yy = np.meshgrid(x0, x1) fig, (ax_l, ax_a) = plot_setup_2d(**kwargs) - cc = cm.get_cmap() + cc = cm.get_cmap('inferno') # Latent function estimate # hf =ax_l.plot_wireframe(xx, yy, np.reshape(ft, (nx, nx)), color=lines[0]) @@ -233,6 +234,7 @@ def estimate_plots2D(xt, ft, mu_t, fhat, vhat, E_y, rel_sigma, # h_yt = ax_a.plot_wireframe(xx, yy, np.reshape(mu_t, (nx, nx)), color=lines[1]) hEy = ax_a.plot_wireframe(xx, yy, np.reshape(E_y, (nx, nx)), color=lines[3]) + cc = cm.get_cmap('Blues') norm_py = p_a_y/p_a_y.max() for y, py in zip(y_samples, norm_py): ax_a.scatter(xt[:, 0], xt[:, 1], y, s=py*15.0, marker='o', c=cc(py)) From 5413feffc2cb91078e66ddce000272b572ee0668 Mon Sep 17 00:00:00 2001 From: Nicholas Lawrance Date: Sat, 24 Mar 2018 18:26:55 +0100 Subject: [PATCH 25/38] Trying to fix regression covariance because broken --- GP_preference_demo.py | 6 ++-- GP_regression_demo.py | 67 +++++++++++++++++++++++++++++++------------ GPpref.py | 4 +-- GPr.py | 8 +++--- plot_tools.py | 4 +-- 5 files changed, 59 insertions(+), 30 deletions(-) diff --git a/GP_preference_demo.py b/GP_preference_demo.py index 79ba404..db084e0 100644 --- a/GP_preference_demo.py +++ b/GP_preference_demo.py @@ -14,7 +14,7 @@ use_test_data = False # test_data.data3 # verbose = 2 -with open('./data/statruns_2D.yaml', 'rt') as fh: +with open('./data/low_freq_2.yaml', 'rt') as fh: wave = yaml.safe_load(fh) try: np.random.seed(wave['statrun_params']['randseed']) @@ -24,10 +24,10 @@ random_wave = test_data.MultiWave(n_dimensions=d_x, **wave['wave_params']) log_hyp = np.log(wave['hyperparameters']) -n_rel_train = 15 +n_rel_train = 0 n_abs_train = 2 -n_xplot = 21 +n_xplot = 101 n_posterior_samples = 3 random_wave.print_values() diff --git a/GP_regression_demo.py b/GP_regression_demo.py index 4580219..55a8f89 100644 --- a/GP_regression_demo.py +++ b/GP_regression_demo.py @@ -3,52 +3,81 @@ import scipy.optimize as op import matplotlib.pyplot as plt import GPr -np.random.seed(0) +import plot_tools as ptt +plt.rc('font',**{'family':'serif','sans-serif':['Computer Modern Roman']}) +plt.rc('text', usetex=True) + +f_sigma = 0.15 +n_samples = 6 +r_seed = 2 +optimise_hp = False # Define polynomial function to be modelled def true_function(x): - c = np.array([3,5,-9,-3,2],float) - y = np.polyval(c,x) + # c = np.array([3,5,-9,-3,2],float) + # y = np.polyval(c,x) + y = np.sin(x*2*np.pi) return y + # Define noisy observation function -def obs_function(x): - y = true_function(x) + np.random.normal(0,0.1,len(x)) +def obs_function(x, sigma): + y = true_function(x) + np.random.normal(0, sigma, len(x)) return y + # Main program # Plot true function -x_plot = np.arange(0,1,0.01,float) -y_plot = true_function(x_plot) +x_plot = np.arange(-0.5, 1.5, 0.01, dtype='float') +fx_plot = true_function(x_plot) plt.figure() -plt.plot(x_plot,y_plot,'k-') +plt.plot(x_plot, fx_plot, 'k-') # Training data -x_train = np.random.random(20) -y_train = obs_function(x_train) +np.random.seed(r_seed) +x_train = 0.6*np.random.random(n_samples)+0.2 +y_train = obs_function(x_train, f_sigma) plt.plot(x_train,y_train,'rx') # Test data -x_test = np.arange(0,1,0.01,float) +x_test = x_plot # GP hyperparameters # note: using log scaling to aid learning hyperparameters with varied magnitudes -log_hyp = np.log([1,1,0.1]) +log_hyp = np.log([0.36, 1.3, 0.2]) mean_hyp = 0 like_hyp = 0 # Initialise GP for hyperparameter training -initGP = GPr.GaussianProcess(log_hyp,mean_hyp,like_hyp,"SE","zero","zero",x_train,y_train) +initGP = GPr.GaussianProcess(log_hyp, mean_hyp, like_hyp, "SE", "zero", "zero", x_train, y_train) # Run optimisation routine to learn hyperparameters -opt_log_hyp = op.fmin(initGP.compute_likelihood,log_hyp) +if optimise_hp: + opt_log_hyp = op.fmin(initGP.compute_likelihood, log_hyp) +else: + opt_log_hyp = log_hyp # Learnt GP with optimised hyperparameters optGP = GPr.GaussianProcess(opt_log_hyp,mean_hyp,like_hyp,"SE","zero","zero",x_train,y_train) -y_test,cov_y = optGP.compute_prediction(x_test) +fhat_test, fhat_var = optGP.compute_prediction(x_test) + +h_fig,h_ax = plt.subplots() +h_fx, patch_fx = ptt.plot_with_bounds(h_ax, x_plot, fx_plot, f_sigma, c=ptt.lines[0]) +h_y, = h_ax.plot(x_train, y_train, 'rx', mew=1.0, ms=8) +h_fhat, patch_fhat = ptt.plot_with_bounds(h_ax, x_plot, fhat_test, np.sqrt(fhat_var.diagonal()), c=ptt.lines[1], ls='--') +# h_pp = h_ax.plot(x_plot, y_post.T, c='grey', ls='--', lw=0.8) + +h_ax.set_ylim([-3, 3]) +gp_str = '$\hat{{f}}(x) \sim \mathcal{{GP}}(l={0:0.2f}, \sigma_f={1:0.2f}, \sigma_n={2:0.2f})$' +gp_l, gp_sigf, gp_sign = optGP.covFun.hyp +gp_str = gp_str.format(gp_l, gp_sigf, gp_sign) +h_ax.legend((h_fx, h_y, h_fhat),('$f(x)$', '$y \sim \mathcal{{N}}(f(x), {0:0.1f}^2)$'.format(f_sigma), gp_str), loc='best') +h_ax.set_xlabel('$x$') +#h_fig.savefig('fig/regression_example.pdf', bbox_inches='tight', transparent='true') + # Plot true and modelled functions -plt.plot(x_test,y_test,'b-') -plt.plot(x_test,y_test+np.sqrt(cov_y),'g--') -plt.plot(x_test,y_test-np.sqrt(cov_y),'g--') -plt.show() +# plt.plot(x_test,y_test,'b-') +# plt.plot(x_test,y_test+np.sqrt(cov_y),'g--') +# plt.plot(x_test,y_test-np.sqrt(cov_y),'g--') +plt.show(block=False) diff --git a/GPpref.py b/GPpref.py index 6664c29..2c873a8 100644 --- a/GPpref.py +++ b/GPpref.py @@ -76,7 +76,7 @@ def compute_Kxz_matrix(self, z): class PrefProbit(object): type = 'preference' y_type = 'discrete' - y_list = np.array([-1, 1], dtype='int') + y_list = np.array([[-1], [1]], dtype='int') def __init__(self, sigma=1.0): self.set_hyper([sigma]) @@ -306,7 +306,7 @@ def generate_samples(self, f): class AbsBoundProbit(object): type = 'bounded continuous' y_type = 'bounded' - y_list = np.linspace(0.01, 0.99, 101) + y_list = np.atleast_2d(np.linspace(0.01, 0.99, 101)).T def __init__(self, sigma=1.0, v=10.0): # v is the precision, kind of related to inverse of noise, high v is sharp distributions diff --git a/GPr.py b/GPr.py index 96be1e3..dbc704e 100644 --- a/GPr.py +++ b/GPr.py @@ -47,10 +47,10 @@ def compute_prediction(self,testInput): Kxx = self.covFun.compute_Kxx_matrix() iKxx = np.linalg.inv(Kxx) Kzx = Kxz.T - K_diag = np.diagonal(np.dot(np.dot(Kzx,iKxx),Kxz)) - K_noise = self.covFun.sf2*np.ones(np.size(testInput,axis=0)) + K_diag = np.dot(np.dot(Kzx,iKxx),Kxz) + K_noise = self.covFun.sn2*np.eye(np.size(testInput,axis=0)) fz = np.dot(np.dot(Kzx,iKxx),self.trainTarget.T) - cov_fz = K_noise - K_diag + cov_fz = K_noise + K_diag return fz, cov_fz # Define GP negative log marginal likelihood function @@ -93,7 +93,7 @@ def __init__(self,logHyp,x): self.hyp = np.exp(self.logHyp) # hyperparameters n = len(self.hyp) self.M = self.hyp[:n-2] # length scales - self.sf2 = self.hyp[n-2]**2 # squared exponential variance + self.sf2 = self.hyp[n-2]**2 # sigma_f variance self.sn2 = self.hyp[n-1]**2 # noise variance def compute_Kxx_matrix(self): diff --git a/plot_tools.py b/plot_tools.py index 8fbd5e1..987e909 100644 --- a/plot_tools.py +++ b/plot_tools.py @@ -28,7 +28,7 @@ def make_poly_array(x,y,sigma): return xy -def plot_with_bounds(ax, x, y, s, c=lines[0]): +def plot_with_bounds(ax, x, y, s, c=lines[0], lw=1.5, *args, **kwargs): isort = np.argsort(x.flat) xx, yy = x[isort], y[isort] try: @@ -37,7 +37,7 @@ def plot_with_bounds(ax, x, y, s, c=lines[0]): ss = s xy = make_poly_array(xx, yy, ss) h_patch = Polygon(xy, ec=c, fc=lighten(c, 3), alpha=0.5) - h_fx, = ax.plot(xx, yy, lw=1.5, c=c) + h_fx, = ax.plot(xx, yy, lw=lw, c=c, *args, **kwargs) ax.add_patch(h_patch) clim = ax.get_ylim() ax.set_ylim(bottom = min(clim[0],xy[:,1].min()), top = max(clim[1], xy[:,1].max())) From 94158661be93c839f562f983ca4bb6a8798115e0 Mon Sep 17 00:00:00 2001 From: Nicholas Lawrance Date: Mon, 26 Mar 2018 18:25:27 +0200 Subject: [PATCH 26/38] Added full cov to GPr, modified plot code slightly --- GP_preference_demo.py | 8 ++--- GP_regression_demo.py | 28 +++++++++------- GPr.py | 42 +++++++++++------------ plot_tools.py | 77 ++++++++++++++++++++++--------------------- 4 files changed, 81 insertions(+), 74 deletions(-) diff --git a/GP_preference_demo.py b/GP_preference_demo.py index db084e0..7b4f1e0 100644 --- a/GP_preference_demo.py +++ b/GP_preference_demo.py @@ -24,8 +24,8 @@ random_wave = test_data.MultiWave(n_dimensions=d_x, **wave['wave_params']) log_hyp = np.log(wave['hyperparameters']) -n_rel_train = 0 -n_abs_train = 2 +n_rel_train = 5 +n_abs_train = 0 n_xplot = 101 n_posterior_samples = 3 @@ -98,7 +98,7 @@ t_r=r'Posterior relative likelihood $P(x_0 \succ x_1 | \mathcal{Y}, \theta)$') p_err = test_data.rel_error(mu_true, p_rel_y_true, E_y, p_rel_y_post, weight=False) print "WRMS: {0:0.3f}, WRMS_MC: {1:0.3f}, p_err: {2:0.3f}".format(wrms, wrms2, p_err) - plt.show() + plt.show(block=False) elif d_x is 2: @@ -113,7 +113,7 @@ t_l=r'$\mathcal{GP}$ latent function estimate $\hat{f}(x)$', t_a=r'Posterior absolute likelihood, $p(u | \mathcal{Y}, \theta)$') print "WRMS: {0:0.3f}, WRMS_MC: {1:0.3f}".format(wrms, wrms2) - plt.show() + plt.show(block=False) else: diff --git a/GP_regression_demo.py b/GP_regression_demo.py index 55a8f89..fba2edf 100644 --- a/GP_regression_demo.py +++ b/GP_regression_demo.py @@ -8,8 +8,9 @@ plt.rc('text', usetex=True) f_sigma = 0.15 -n_samples = 6 -r_seed = 2 +n_samples = 10 +r_seed = 1 +n_posterior_samples = 3 optimise_hp = False # Define polynomial function to be modelled @@ -22,29 +23,30 @@ def true_function(x): # Define noisy observation function def obs_function(x, sigma): - y = true_function(x) + np.random.normal(0, sigma, len(x)) + y = true_function(x) + np.random.normal(0, sigma, size=x.shape) return y # Main program # Plot true function -x_plot = np.arange(-0.5, 1.5, 0.01, dtype='float') +x_plot = np.atleast_2d(np.arange(-0.5, 1.5, 0.01, dtype='float')).T fx_plot = true_function(x_plot) plt.figure() plt.plot(x_plot, fx_plot, 'k-') # Training data np.random.seed(r_seed) -x_train = 0.6*np.random.random(n_samples)+0.2 +x_train = np.atleast_2d(0.6*np.random.random(n_samples)+0.2).T +# x_train = np.array([-10.0]) y_train = obs_function(x_train, f_sigma) -plt.plot(x_train,y_train,'rx') +plt.plot(x_train, y_train, 'rx') # Test data x_test = x_plot # GP hyperparameters # note: using log scaling to aid learning hyperparameters with varied magnitudes -log_hyp = np.log([0.36, 1.3, 0.2]) +log_hyp = np.log([0.3, 1.0, 0.2]) mean_hyp = 0 like_hyp = 0 @@ -57,18 +59,22 @@ def obs_function(x, sigma): else: opt_log_hyp = log_hyp -# Learnt GP with optimised hyperparameters +# Learned GP with optimised hyperparameters optGP = GPr.GaussianProcess(opt_log_hyp,mean_hyp,like_hyp,"SE","zero","zero",x_train,y_train) fhat_test, fhat_var = optGP.compute_prediction(x_test) h_fig,h_ax = plt.subplots() h_fx, patch_fx = ptt.plot_with_bounds(h_ax, x_plot, fx_plot, f_sigma, c=ptt.lines[0]) h_y, = h_ax.plot(x_train, y_train, 'rx', mew=1.0, ms=8) -h_fhat, patch_fhat = ptt.plot_with_bounds(h_ax, x_plot, fhat_test, np.sqrt(fhat_var.diagonal()), c=ptt.lines[1], ls='--') +h_fhat, patch_fhat = ptt.plot_with_bounds(h_ax, x_plot, fhat_test, np.sqrt(fhat_var.diagonal())[np.newaxis].T, c=ptt.lines[1], ls='--') +f_post = np.random.multivariate_normal(fhat_test.flatten(), fhat_var, n_posterior_samples) +h_pp = h_ax.plot(x_plot, f_post.T, '--', color='grey', lw=0.8) +h_ax.autoscale() + # h_pp = h_ax.plot(x_plot, y_post.T, c='grey', ls='--', lw=0.8) -h_ax.set_ylim([-3, 3]) -gp_str = '$\hat{{f}}(x) \sim \mathcal{{GP}}(l={0:0.2f}, \sigma_f={1:0.2f}, \sigma_n={2:0.2f})$' +# h_ax.set_ylim([-3, 3]) +gp_str = '$\hat{{f}}(x)|\mathcal{{Y}} \sim \mathcal{{GP}}(\mathbf{{0}}, K_{{SE}}:l={0:0.2f}, \sigma_f={1:0.2f}, \sigma_n={2:0.2f})$' gp_l, gp_sigf, gp_sign = optGP.covFun.hyp gp_str = gp_str.format(gp_l, gp_sigf, gp_sign) h_ax.legend((h_fx, h_y, h_fhat),('$f(x)$', '$y \sim \mathcal{{N}}(f(x), {0:0.1f}^2)$'.format(f_sigma), gp_str), loc='best') diff --git a/GPr.py b/GPr.py index dbc704e..93e6a43 100644 --- a/GPr.py +++ b/GPr.py @@ -43,23 +43,23 @@ def __init__(self,log_hyp,mean_hyp,like_hyp,covFunName,meanFunName,likeFunName, # Define GP prediction function def compute_prediction(self,testInput): - Kxz = self.covFun.compute_Kxz_matrix(testInput) - Kxx = self.covFun.compute_Kxx_matrix() - iKxx = np.linalg.inv(Kxx) + Kxx = self.covFun.compute_K_matrix() + Kxz = self.covFun.compute_K_matrix(X1=testInput) + Kzz = self.covFun.compute_K_matrix(testInput, testInput) + iKxx = np.linalg.inv(Kxx + self.covFun.sn2*np.eye(Kxx.shape[0])) Kzx = Kxz.T K_diag = np.dot(np.dot(Kzx,iKxx),Kxz) - K_noise = self.covFun.sn2*np.eye(np.size(testInput,axis=0)) - fz = np.dot(np.dot(Kzx,iKxx),self.trainTarget.T) - cov_fz = K_noise + K_diag + fz = np.dot(np.dot(Kzx,iKxx), self.trainTarget) + cov_fz = Kzz - K_diag return fz, cov_fz # Define GP negative log marginal likelihood function - def compute_likelihood(self,hyp): + def compute_likelihood(self, hyp): n = np.size(self.trainInput,axis=0) - covSE = SquaredExponential(hyp,self.trainInput) - Kxx = covSE.compute_Kxx_matrix() + covSE = SquaredExponential(hyp, self.trainInput) + Kxx = covSE.compute_K_matrix() m = self.meanFun.y - L = np.linalg.cholesky(Kxx) + L = np.linalg.cholesky(Kxx +self.covFun.sn2*np.eye(Kxx.shape[0])) iKxx = np.linalg.solve(L.T,np.linalg.solve(L,np.eye(n))) y = np.reshape(self.trainTarget,(len(self.trainTarget),1)) err_y = np.dot(np.dot((y-m).T,iKxx),(y-m))/2 @@ -95,16 +95,14 @@ def __init__(self,logHyp,x): self.M = self.hyp[:n-2] # length scales self.sf2 = self.hyp[n-2]**2 # sigma_f variance self.sn2 = self.hyp[n-1]**2 # noise variance - - def compute_Kxx_matrix(self): - scaledX = self.x/self.M - sqDist = squared_distance(scaledX,scaledX) - Kxx = self.sn2*np.eye(np.size(self.x,axis=0))+self.sf2*np.exp(-0.5*sqDist) - return Kxx - - def compute_Kxz_matrix(self,z): - scaledX = self.x/self.M - scaledZ = z/self.M + + def compute_K_matrix(self, X0=None, X1=None): + if X0 is None: + X0 = self.x + if X1 is None: + X1 = self.x + scaledX = X0/self.M + scaledZ = X1/self.M sqDist = squared_distance(scaledX,scaledZ) - Kxz = self.sf2*np.exp(-0.5*sqDist) - return Kxz \ No newline at end of file + K = self.sf2*np.exp(-0.5*sqDist) + return K diff --git a/plot_tools.py b/plot_tools.py index 987e909..6d66978 100644 --- a/plot_tools.py +++ b/plot_tools.py @@ -6,19 +6,21 @@ from nice_plot_colors import * from cycler import cycler from mpl_toolkits.mplot3d.art3d import Line3DCollection - # plt.rc('axes', prop_cycle=(cycler('color', [greyify(c, .5, .8) for c in reversed(lines)]))) plt.rc('axes', prop_cycle=(cycler('color', lines))) + def xgen(xp, n_dim): for i in range(n_dim): yield xp + def make_meshlist(x_plot, d_x): xx = xgen(x_plot, d_x) x_mesh = np.vstack(np.meshgrid(*xx, copy=False)).reshape(d_x, -1).T return x_mesh + def make_poly_array(x,y,sigma): nx = len(x) sigma = np.atleast_2d(sigma) @@ -65,11 +67,39 @@ def plot_setup_1d(t_l = r'Latent function, $f(x)$', t_a = r'Absolute likelihood, ax_r.set_ylabel('$x_1$') return fig, (ax_l, ax_a, ax_r) -def plot_relative_likelihood(ax, p_y, extent): + +def plot_relative_queries(ax, uvr_train, fuvr_train, yr_train, class_icons, marker_options=None): + for uv, fuv, y in zip(uvr_train, fuvr_train, yr_train): + ax.plot(uv, fuv, 'b-', color=lighten(lines[0])) + ax.plot(uv[(y + 1) / 2], fuv[(y + 1) / 2], class_icons[(y[0] + 1) / 2], **marker_options) + + +def plot_absolute_likelihood(ax, p_a_y, xt, mu_t, y_samples, E_y=None, xa_train=None, ya_train=None): + dely = y_samples[1, 0] - y_samples[0, 0] + abs_extent = [xt[0, 0], xt[-1, 0], y_samples[0, 0] - 0.5*dely, y_samples[-1, 0] + 0.5*dely] + vmax = max(1.0, p_a_y.max()) + h_pat = ax.imshow(p_a_y, origin='lower', extent=abs_extent, aspect='auto', vmin=0.0, vmax=vmax) + if xa_train is not None and xa_train.shape[0] > 0: + ax.plot(xa_train, ya_train, 'w+') + h_lines = ax.plot(xt, mu_t, c=lines[0]) + legend_entries = [r'True mean, $E[y]$'] + if E_y is not None: + h_lines.extend(ax.plot(xt, E_y, color=lines[3])) + legend_entries.append(r'Posterior mean, $E_{p(y|\mathcal{Y})}\left[y\right]$') + ax.legend(h_lines, legend_entries) + ax.set_xlim(xt[0], xt[-1]) + ax.get_figure().colorbar(h_pat, ax=ax) + return h_pat + +def plot_relative_likelihood(ax, p_y, extent, uvr_train=None, yr_train=None, class_icons=None, marker_options=None): h_p = ax.imshow(p_y, origin='lower', extent=extent, vmin=0.0, vmax=1.0) h_pc = ax.contour(p_y, levels=[0.5], origin='lower', linewidths=2, extent=extent) plt.clabel(h_pc, inline=1, fontsize=10) ax.get_figure().colorbar(h_p, ax=ax) + + if uvr_train is not None: # and xt.shape[0] > 0: + for uv, y in zip(uvr_train, yr_train): + ax.plot(uv[0], uv[1], class_icons[(y[0] + 1) / 2], **marker_options) return h_p def true_plots(xt, ft, mu_t, rel_sigma, y_samples, p_a_y, p_r_y, xa_train=None, ya_train=None, uvr_train=None, fuvr_train=None, yr_train=None, @@ -81,27 +111,15 @@ def true_plots(xt, ft, mu_t, rel_sigma, y_samples, p_a_y, p_r_y, xa_train=None, # True latent plot_with_bounds(ax_l, xt, ft, rel_sigma, c=lines[0]) + if uvr_train is not None: + plot_relative_queries(ax_l, uvr_train, fuvr_train, yr_train, class_icons, marker_options) # True absolute likelihood - dely = y_samples[1, 0] - y_samples[0, 0] - abs_extent = [xt[0, 0], xt[-1, 0], y_samples[0, 0] - 0.5*dely, y_samples[-1, 0] + 0.5*dely] - vmax = max(1.0, p_a_y.max()) - h_pat = ax_a.imshow(p_a_y, origin='lower', extent=abs_extent, aspect='auto', vmin=0.0, vmax=vmax) - if xa_train is not None and xa_train.shape[0] > 0: - ax_a.plot(xa_train, ya_train, 'w+') - h_yt, = ax_a.plot(xt, mu_t, c=lines[0]) - ax_a.legend([h_yt], ['$E[y]$']) - ax_a.set_xlim(xt[0], xt[-1]) - fig.colorbar(h_pat, ax=ax_a) + h_pat = plot_absolute_likelihood(ax_a, p_a_y, xt, mu_t, y_samples, xa_train=xa_train, ya_train=ya_train) # True relative likelihood rel_y_extent = [xt[0, 0], xt[-1, 0], xt[0, 0], xt[-1, 0]] - h_prt = plot_relative_likelihood(ax_r, p_r_y, extent=rel_y_extent) - if uvr_train is not None and xt.shape[0] > 0: - for uv, fuv, y in zip(uvr_train, fuvr_train, yr_train): - ax_r.plot(uv[0], uv[1], class_icons[(y[0] + 1) / 2], **marker_options) - ax_l.plot(uv, fuv, 'b-', color=lighten(lines[0])) - ax_l.plot(uv[(y + 1) / 2], fuv[(y + 1) / 2], class_icons[(y[0] + 1) / 2], **marker_options) + h_prt = plot_relative_likelihood(ax_r, p_r_y, rel_y_extent, uvr_train, yr_train, class_icons, marker_options) return fig, (ax_l, ax_a, ax_r) def estimate_plots(xt, ft, mu_t, fhat, vhat, E_y, rel_sigma, @@ -116,33 +134,18 @@ def estimate_plots(xt, ft, mu_t, fhat, vhat, E_y, rel_sigma, # Latent function hf, hpf = plot_with_bounds(ax_l, xt, ft, rel_sigma, c=lines[0]) - hf_hat, hpf_hat = plot_with_bounds(ax_l, xt, fhat, np.sqrt(np.atleast_2d(vhat.diagonal()).T), c=lines[1]) + if uvr_train is not None: + plot_relative_queries(ax_l, uvr_train, fuvr_train, yr_train, class_icons, marker_options) ax_l.legend([hf, hf_hat], [r'True latent function, $f(x)$', r'$\mathcal{GP}$ estimate $\hat{f}(x)$']) # Absolute posterior likelihood - dely = y_samples[1, 0]-y_samples[0, 0] - abs_extent = [xt[0, 0], xt[-1, 0], y_samples[0, 0]-0.5*dely, y_samples[-1, 0]+0.5*dely] - vmax = max(1.0, p_a_y.max()) - h_pap = ax_a.imshow(p_a_y, origin='lower', extent=abs_extent, aspect='auto') - h_yt, = ax_a.plot(xt, mu_t, c=lines[0]) - hEy, = ax_a.plot(xt, E_y, color=lines[3]) - if xa_train.shape[0] > 0: - ax_a.plot(xa_train, ya_train, 'w+') - ax_a.set_xlim(xt[0], xt[-1]) - ax_a.legend([h_yt, hEy], - [r'True mean, $E[y]$', r'Posterior mean, $E_{p(y|\mathcal{Y})}\left[y\right]$']) - fig.colorbar(h_pap, ax=ax_a) + plot_absolute_likelihood(ax_a, p_a_y, xt, mu_t, y_samples, E_y=E_y, xa_train=xa_train, ya_train=xa_train) # Relative posterior likelihood rel_y_extent = [xt[0, 0], xt[-1, 0], xt[0, 0], xt[-1, 0]] - h_prp = plot_relative_likelihood(ax_r, p_r_y, extent=rel_y_extent) - if uvr_train.shape[0] > 0: - for uv, fuv, y in zip(uvr_train, fuvr_train, yr_train): - ax_r.plot(uv[0], uv[1], class_icons[(y[0] + 1) / 2], **marker_options) - ax_l.plot(uv, fuv, 'b-', color=lighten(lines[0])) - ax_l.plot(uv[(y + 1) / 2], fuv[(y + 1) / 2], class_icons[(y[0] + 1) / 2], **marker_options) + h_prp = plot_relative_likelihood(ax_r, p_r_y, rel_y_extent, uvr_train, yr_train, class_icons, marker_options) return fig, (ax_l, ax_a, ax_r) From 334e74a0e54701b1a4b92afbb7b717a921bef8af Mon Sep 17 00:00:00 2001 From: Nicholas Lawrance Date: Mon, 26 Mar 2018 23:50:15 +0200 Subject: [PATCH 27/38] Plot kwargs for posterior samples --- GP_preference_demo.py | 1 + plot_tools.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/GP_preference_demo.py b/GP_preference_demo.py index 7b4f1e0..474d882 100644 --- a/GP_preference_demo.py +++ b/GP_preference_demo.py @@ -94,6 +94,7 @@ ptt.estimate_plots(x_test, f_true, mu_true, fhat, vhat, E_y, wave['rel_obs_params']['sigma'], abs_y_samples, p_abs_y_post, p_rel_y_post, x_abs, y_abs, x_rel[uvi_rel][:,:,0], fuv_rel, y_rel, n_posterior_samples=n_posterior_samples, + posterior_plot_kwargs={'color':'grey', 'ls':'--'}, t_a=r'Posterior absolute likelihood, $p(u | \mathcal{Y}, \theta)$', t_r=r'Posterior relative likelihood $P(x_0 \succ x_1 | \mathcal{Y}, \theta)$') p_err = test_data.rel_error(mu_true, p_rel_y_true, E_y, p_rel_y_post, weight=False) diff --git a/plot_tools.py b/plot_tools.py index 6d66978..670fe7e 100644 --- a/plot_tools.py +++ b/plot_tools.py @@ -124,13 +124,14 @@ def true_plots(xt, ft, mu_t, rel_sigma, y_samples, p_a_y, p_r_y, xa_train=None, def estimate_plots(xt, ft, mu_t, fhat, vhat, E_y, rel_sigma, y_samples, p_a_y, p_r_y, xa_train, ya_train, uvr_train, fuvr_train, yr_train, - class_icons = ['ko', 'wo'], marker_options = {'mec':'k', 'mew':0.5}, n_posterior_samples=0, *args, **kwargs): + class_icons = ['ko', 'wo'], marker_options = {'mec':'k', 'mew':0.5}, n_posterior_samples=0, + posterior_plot_kwargs={}, **kwargs): fig, (ax_l, ax_a, ax_r) = plot_setup_1d(**kwargs) # Posterior samples if n_posterior_samples > 0: f_post = np.random.multivariate_normal(fhat.flatten(), vhat, n_posterior_samples) - h_pp = ax_l.plot(xt, f_post.T, lw=0.8) + h_pp = ax_l.plot(xt, f_post.T, lw=0.8, **posterior_plot_kwargs) # Latent function hf, hpf = plot_with_bounds(ax_l, xt, ft, rel_sigma, c=lines[0]) From 8716024ccf4e7323c7565be331a20b8342948109 Mon Sep 17 00:00:00 2001 From: Nicholas Lawrance Date: Tue, 12 Jun 2018 18:08:06 +0200 Subject: [PATCH 28/38] Added 2D support for preference demo and statruns --- GP_preference_demo.py | 44 ++++++++++++++++----------------------- active_learners.py | 18 +++++++++------- active_statruns.py | 17 ++++++++------- plot_tools.py | 21 ++++++++++++------- pref_active_learning.py | 46 ++++++++++++++++++++--------------------- test_data.py | 10 ++++----- 6 files changed, 79 insertions(+), 77 deletions(-) diff --git a/GP_preference_demo.py b/GP_preference_demo.py index 474d882..c685c77 100644 --- a/GP_preference_demo.py +++ b/GP_preference_demo.py @@ -1,4 +1,4 @@ -# Simple 1D GP classification example +# Simple example showing GP estimate with relative and/or absolute observations import numpy as np import matplotlib.pyplot as plt import GPpref @@ -14,7 +14,7 @@ use_test_data = False # test_data.data3 # verbose = 2 -with open('./data/low_freq_2.yaml', 'rt') as fh: +with open('./data/statruns_jan2018.yaml', 'rt') as fh: wave = yaml.safe_load(fh) try: np.random.seed(wave['statrun_params']['randseed']) @@ -25,7 +25,7 @@ log_hyp = np.log(wave['hyperparameters']) n_rel_train = 5 -n_abs_train = 0 +n_abs_train = 5 n_xplot = 101 n_posterior_samples = 3 @@ -45,8 +45,6 @@ mu_true = abs_obs_fun.mean_link(x_test) abs_y_samples = abs_obs_fun.l.y_list p_abs_y_true = abs_obs_fun.observation_likelihood_array(x_test, abs_y_samples) -if d_x is 1: - p_rel_y_true = rel_obs_fun.observation_likelihood_array(x_test) # Training data - this is a bit weird, but we sample x values, then the uv pairs # are actually indexes into x, because it is easier computationally. You can @@ -82,41 +80,35 @@ wrms2 = test_data.wrms_misclass(mu_true, E_y) if d_x is 1: + p_rel_y_true = rel_obs_fun.observation_likelihood_array(x_test) p_rel_y_post = prefGP.rel_posterior_likelihood_array(fhat=fhat, varhat=vhat) +else: + p_rel_y_true, p_rel_y_post = None, None +if d_x <= 2: # Plot true functions - fig_t, (ax_t_l, ax_t_a, ax_t_r) = ptt.true_plots(x_test, f_true, mu_true, wave['rel_obs_params']['sigma'], - abs_y_samples, p_abs_y_true, p_rel_y_true, - t_l=r'True latent function, $f(x)$') + fig_t, ax_t = ptt.true_plots(x_test, f_true, mu_true, wave['rel_obs_params']['sigma'], + abs_y_samples, p_abs_y_true, p_rel_y_true, + t_l=r'True latent function, $f(x)$') # Posterior estimates - fig_p, (ax_p_l, ax_p_a, ax_p_r) = \ + fig_p, ax_p = \ ptt.estimate_plots(x_test, f_true, mu_true, fhat, vhat, E_y, wave['rel_obs_params']['sigma'], abs_y_samples, p_abs_y_post, p_rel_y_post, - x_abs, y_abs, x_rel[uvi_rel][:,:,0], fuv_rel, y_rel, n_posterior_samples=n_posterior_samples, + x_abs, y_abs, x_rel[uvi_rel], fuv_rel, y_rel, n_posterior_samples=n_posterior_samples, posterior_plot_kwargs={'color':'grey', 'ls':'--'}, + t_l=r'$\mathcal{GP}$ latent function estimate $\hat{f}(x)$', t_a=r'Posterior absolute likelihood, $p(u | \mathcal{Y}, \theta)$', t_r=r'Posterior relative likelihood $P(x_0 \succ x_1 | \mathcal{Y}, \theta)$') - p_err = test_data.rel_error(mu_true, p_rel_y_true, E_y, p_rel_y_post, weight=False) - print "WRMS: {0:0.3f}, WRMS_MC: {1:0.3f}, p_err: {2:0.3f}".format(wrms, wrms2, p_err) - plt.show(block=False) + if d_x == 1: + p_err = test_data.rel_error(mu_true, p_rel_y_true, E_y, p_rel_y_post, weight=False) + print "WRMS: {0:0.3f}, WRMS_MC: {1:0.3f}, p_err: {2:0.3f}".format(wrms, wrms2, p_err) + else: + print "WRMS: {0:0.3f}, WRMS_MC: {1:0.3f}".format(wrms, wrms2) -elif d_x is 2: - # Plot true functions - fig_t, (ax_t_l, ax_t_a) = ptt.true_plots2D(x_test, f_true, mu_true, wave['rel_obs_params']['sigma'], - abs_y_samples, p_abs_y_true, t_l=r'True latent function, $f(x)$') - - # Posterior estimates - fig_p, (ax_p_l, ax_p_a) = \ - ptt.estimate_plots2D(x_test, f_true, mu_true, fhat, vhat, E_y, wave['rel_obs_params']['sigma'], - abs_y_samples, p_abs_y_post, x_abs, y_abs, x_rel[uvi_rel], fuv_rel, y_rel, - t_l=r'$\mathcal{GP}$ latent function estimate $\hat{f}(x)$', - t_a=r'Posterior absolute likelihood, $p(u | \mathcal{Y}, \theta)$') - print "WRMS: {0:0.3f}, WRMS_MC: {1:0.3f}".format(wrms, wrms2) plt.show(block=False) - else: print "Input state space dimension: {0} is too high to plot".format(d_x) print "WRMS: {0:0.3f}, WRMS_MC: {1:0.3f}".format(wrms, wrms2) diff --git a/active_learners.py b/active_learners.py index 909d339..598dcc6 100644 --- a/active_learners.py +++ b/active_learners.py @@ -64,23 +64,27 @@ def linear_domain_sampler(self, n_samples, domain=None): x_test = x_test*np.diff(domain, axis=0) + domain[0, :] return x_test - def create_posterior_plot(self, x_test, f_true, mu_true, rel_sigma, fuv_train, abs_y_samples): + def create_posterior_plot(self, x_test, f_true, mu_true, rel_sigma, fuv_train, abs_y_samples, **plot_kwargs): # Latent predictions + d_x = x_test.shape[1] fhat, vhat = self.predict_latent(x_test) - p_abs_y_post, E_y = self.abs_posterior_likelihood(abs_y_samples, fhat=fhat, varhat=vhat) - p_rel_y_post = self.rel_posterior_likelihood_array(fhat=fhat, varhat=vhat) + if d_x is 1: + p_rel_y_post = self.rel_posterior_likelihood_array(fhat=fhat, varhat=vhat) + else: + p_rel_y_post = None x_train, uvi_train, x_abs_train, y_train, y_abs_train = self.get_observations() - uv_train = x_train[uvi_train][:, :, 0] + uv_train = x_train[uvi_train] # Posterior estimates - fig_p, (ax_p_l, ax_p_a, ax_p_r) = \ + fig_p, ax_p = \ ptt.estimate_plots(x_test, f_true, mu_true, fhat, vhat, E_y, rel_sigma, abs_y_samples, p_abs_y_post, p_rel_y_post, x_abs_train, y_abs_train, uv_train, fuv_train, y_train, + t_l=r'$\mathcal{GP}$ latent function estimate $\hat{f}(x)$', t_a=r'Posterior absolute likelihood, $p(y | \mathcal{Y}, \theta)$', - t_r=r'Posterior relative likelihood $P(x_0 \succ x_1 | \mathcal{Y}, \theta)$') - return fig_p, (ax_p_l, ax_p_a, ax_p_r) + t_r=r'Posterior relative likelihood $P(x_0 \succ x_1 | \mathcal{Y}, \theta)$', **plot_kwargs) + return fig_p, ax_p class MaxVar(ActiveLearner): diff --git a/active_statruns.py b/active_statruns.py index 2d85b1e..c58644f 100644 --- a/active_statruns.py +++ b/active_statruns.py @@ -18,7 +18,7 @@ parser = argparse.ArgumentParser(description='Statruns for active learning with preference GP') parser.add_argument('-np', '--no-plots', dest='make_plots', action='store_false', help='Turn off plots (default False)') -parser.add_argument('yaml_config', default=yaml_config, help='YAML config file') +parser.add_argument('-y', '--yaml-config', default=yaml_config, help='YAML config file') args = parser.parse_args() @@ -44,19 +44,20 @@ n_rel_samples = run_parameters['statrun_params']['n_rel_samples'] # Define polynomial function to be modelled -random_wave = test_data.MultiWave(**run_parameters['wave_params']) +d_x = run_parameters['GP_params']['hyper_counts'][0]-1 +random_wave = test_data.MultiWave(n_dimensions=d_x, **run_parameters['wave_params']) now_time = time.strftime("%Y_%m_%d-%H_%M") data_dir = 'data/' + now_time + '/' ptt.ensure_dir(data_dir) print "Data will be saved to: {0}".format(data_dir) -waver = test_data.WaveSaver(n_trials, random_wave.n_components) +waver = test_data.WaveSaver(n_trials, random_wave.n_components, n_dim=d_x) # True function x_plot = np.linspace(0.0, 1.0, statrun_params['n_xtest'], dtype='float') -x_test = np.atleast_2d(x_plot).T -far_domain = np.array([[-3.0], [-2.0]]) +x_test = ptt.make_meshlist(x_plot, d_x) +far_domain = np.tile(np.array([[-3.0], [-2.0]]), d_x) # Construct active learner objects n_learners = len(run_parameters['learners']) @@ -102,9 +103,9 @@ p_rel_y_true = rel_obs_fun.observation_likelihood_array(x_test) # Initial data - x_rel, uvi_rel, uv_rel, y_rel, fuv_rel = rel_obs_fun.generate_n_observations(statrun_params['n_rel_train'], - n_xdim=1, domain=far_domain) - x_abs, y_abs, mu_abs = abs_obs_fun.generate_n_observations(statrun_params['n_abs_train'], n_xdim=1, + x_rel, uvi_rel, y_rel, fuv_rel = rel_obs_fun.generate_n_observations(statrun_params['n_rel_train'], + n_xdim=d_x, domain=far_domain) + x_abs, y_abs, mu_abs = abs_obs_fun.generate_n_observations(statrun_params['n_abs_train'], n_xdim=d_x, domain=far_domain) model_kwargs = {'x_rel': x_rel, 'uvi_rel': uvi_rel, 'x_abs': x_abs, 'y_rel': y_rel, 'y_abs': y_abs, 'rel_kwargs': run_parameters['rel_obs_params'], 'abs_kwargs': run_parameters['abs_obs_params']} diff --git a/plot_tools.py b/plot_tools.py index 670fe7e..639ce5d 100644 --- a/plot_tools.py +++ b/plot_tools.py @@ -46,9 +46,9 @@ def plot_with_bounds(ax, x, y, s, c=lines[0], lw=1.5, *args, **kwargs): return h_fx, h_patch def plot_setup_1d(t_l = r'Latent function, $f(x)$', t_a = r'Absolute likelihood, $p(y | f(x))$', - t_r = r'Relative likelihood, $P(x_0 \succ x_1 | f(x_0), f(x_1))$'): + t_r = r'Relative likelihood, $P(x_0 \succ x_1 | f(x_0), f(x_1))$', **kwargs): - fig, (ax_l, ax_a, ax_r) = plt.subplots(1, 3) + fig, (ax_l, ax_a, ax_r) = plt.subplots(1, 3, **kwargs) fig.set_size_inches(14.7, 3.5) # Latent function @@ -102,9 +102,11 @@ def plot_relative_likelihood(ax, p_y, extent, uvr_train=None, yr_train=None, cla ax.plot(uv[0], uv[1], class_icons[(y[0] + 1) / 2], **marker_options) return h_p -def true_plots(xt, ft, mu_t, rel_sigma, y_samples, p_a_y, p_r_y, xa_train=None, ya_train=None, uvr_train=None, fuvr_train=None, yr_train=None, +def true_plots(xt, ft, mu_t, rel_sigma, y_samples, p_a_y, p_r_y=None, xa_train=None, ya_train=None, uvr_train=None, fuvr_train=None, yr_train=None, class_icons=['ko', 'wo'], marker_options={'mec':'k', 'mew':0.5}, *args, **kwargs): - + if p_r_y is None: + return true_plots2D(xt, ft, mu_t, rel_sigma, y_samples, p_a_y, xa_train, ya_train, uvr_train, fuvr_train, yr_train, + class_icons, marker_options, *args, **kwargs) # Plot true function, likelihoods and observations fig, (ax_l, ax_a, ax_r) = plot_setup_1d(**kwargs) @@ -126,6 +128,11 @@ def estimate_plots(xt, ft, mu_t, fhat, vhat, E_y, rel_sigma, y_samples, p_a_y, p_r_y, xa_train, ya_train, uvr_train, fuvr_train, yr_train, class_icons = ['ko', 'wo'], marker_options = {'mec':'k', 'mew':0.5}, n_posterior_samples=0, posterior_plot_kwargs={}, **kwargs): + if p_r_y is None: + return estimate_plots2D(xt, ft, mu_t, fhat, vhat, E_y, rel_sigma, + y_samples, p_a_y, xa_train, ya_train, uvr_train, fuvr_train, yr_train, + class_icons, marker_options, **kwargs) + fig, (ax_l, ax_a, ax_r) = plot_setup_1d(**kwargs) # Posterior samples @@ -152,11 +159,11 @@ def estimate_plots(xt, ft, mu_t, fhat, vhat, E_y, rel_sigma, # 2D Plot tools -def plot_setup_2d(t_l = r'Latent function, $f(x)$', t_a = r'Absolute likelihood, $p(y | f(x))$'): +def plot_setup_2d(t_l = r'Latent function, $f(x)$', t_a = r'Absolute likelihood, $p(y | f(x))$', t_r = None, **kwargs): fig = plt.figure() - ax_l = fig.add_subplot(121, projection='3d') - ax_a = fig.add_subplot(122, projection='3d') + ax_l = fig.add_subplot(121, projection='3d', **kwargs) + ax_a = fig.add_subplot(122, projection='3d', **kwargs) fig.set_size_inches(10.0, 3.5) # Latent function diff --git a/pref_active_learning.py b/pref_active_learning.py index 52c259f..ffbfcca 100644 --- a/pref_active_learning.py +++ b/pref_active_learning.py @@ -15,7 +15,7 @@ save_plots = False -with open('./data/statruns_nov2017.yaml', 'rt') as fh: +with open('./data/shortrun_2D.yaml', 'rt') as fh: wave = yaml.safe_load(fh) try: @@ -28,14 +28,15 @@ n_abs_train = 0 n_queries = 20 -n_xplot = 101 +n_xplot = 31 keep_f = True -learner_index = -1 +learner_index = -2 log_hyp = np.log(wave['hyperparameters']) # Define polynomial function to be modelled -random_wave = test_data.MultiWave(**wave['wave_params']) +d_x = wave['GP_params']['hyper_counts'][0]-1 +random_wave = test_data.MultiWave(n_dimensions=d_x, **wave['wave_params']) true_function = random_wave.out random_wave.print_values() @@ -52,32 +53,32 @@ # True function x_plot = np.linspace(0.0,1.0,n_xplot,dtype='float') -x_test = np.atleast_2d(x_plot).T +x_test = ptt.make_meshlist(x_plot, d_x) f_true = abs_obs_fun.f(x_test) mu_true = abs_obs_fun.mean_link(x_test) abs_y_samples = abs_obs_fun.l.y_list p_abs_y_true = abs_obs_fun.observation_likelihood_array(x_test, abs_y_samples) -p_rel_y_true = rel_obs_fun.observation_likelihood_array(x_test) +if d_x is 1: + p_rel_y_true = rel_obs_fun.observation_likelihood_array(x_test) +else: + p_rel_y_true = None # Training data - note the shifted domain to get the sample out of the way -far_domain = np.array([[-3.0], [-2.0]]) -x_rel, uvi_rel, uv_rel, y_rel, fuv_rel = rel_obs_fun.generate_n_observations(n_rel_train, n_xdim=1, domain=far_domain) -x_abs, y_abs, mu_abs = abs_obs_fun.generate_n_observations(n_abs_train, n_xdim=1, domain=far_domain) +far_domain = np.tile(np.array([[-3.0], [-2.0]]), d_x) +x_rel, uvi_rel, y_rel, fuv_rel = rel_obs_fun.generate_n_observations(n_rel_train, n_xdim=d_x, domain=far_domain) +x_abs, y_abs, mu_abs = abs_obs_fun.generate_n_observations(n_abs_train, n_xdim=d_x, domain=far_domain) # Plot true functions -fig_t, (ax_t_l, ax_t_a, ax_t_r) = ptt.true_plots(x_test, f_true, mu_true, rel_sigma, +plot_kwargs = {'xlim':[0.0, 1.0], 'ylim':[0.0, 1.0]} +fig_t, ax_t = ptt.true_plots(x_test, f_true, mu_true, rel_sigma, abs_y_samples, p_abs_y_true, p_rel_y_true, - x_abs, y_abs, uv_rel, fuv_rel, y_rel, - t_l=r'True latent function, $f(x)$') + x_abs, y_abs, x_rel[uvi_rel], fuv_rel, y_rel, + t_l=r'True latent function, $f(x)$', **plot_kwargs) if save_plots: fig_t.savefig(fig_dir+'true.pdf', bbox_inches='tight') # Construct active learner object - -# Construct GP object -prefGP = GPpref.PreferenceGaussianProcess(x_rel, uvi_rel, x_abs, y_rel, y_abs, **wave['GP_params']) - model_kwargs = {'x_rel':x_rel, 'uvi_rel':uvi_rel, 'x_abs':x_abs, 'y_rel':y_rel, 'y_abs':y_abs, 'rel_kwargs': wave['rel_obs_params'], 'abs_kwargs': wave['abs_obs_params']} model_kwargs.update(wave['GP_params']) @@ -90,8 +91,7 @@ f = learner.model.solve_laplace() learner.model.print_hyperparameters() -fig_p, (ax_p_l, ax_p_a, ax_p_r) = learner.model.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, - abs_y_samples) +fig_p, ax_p = learner.model.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, abs_y_samples, **plot_kwargs) if save_plots: wave['learners'] = [learner_kwargs] with open(fig_dir+'params.yaml', 'wt') as fh: @@ -111,7 +111,7 @@ if next_x.shape[0] == 1: next_y, next_f = abs_obs_fun.generate_observations(next_x) learner.model.add_observations(next_x, next_y, keep_f=keep_f) - print 'Abs: x:{0}, y:{1}'.format(next_x[0], next_y[0]), + print '{n:02} - Abs: x:{0}, y:{1}'.format(next_x[0], next_y[0], n=obs_num+1), else: next_y, next_uvi, next_fx = rel_obs_fun.gaussian_multi_pairwise_sampler(next_x) next_fuv = next_fx[next_uvi][:, :, 0] @@ -122,8 +122,7 @@ f = learner.model.solve_laplace() if save_plots: - fig_p, (ax_p_l, ax_p_a, ax_p_r) = learner.model.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, - abs_y_samples) + fig_p, ax_p = learner.model.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, abs_y_samples, **plot_kwargs) pdf_pages.savefig(fig_p, bbox_inches='tight') # fig_p.savefig(fig_dir+'posterior{0:02d}.pdf'.format(obs_num+1), bbox_inches='tight') plt.close(fig_p) @@ -133,10 +132,9 @@ if save_plots: pdf_pages.close() else: - fig_post, (ax_p_l, ax_p_a, ax_p_r) = learner.model.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, - abs_y_samples) + fig_post, ax_p = learner.model.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, abs_y_samples, **plot_kwargs) -plt.show() +plt.show(block=False) print "Finished!" ## SCRAP diff --git a/test_data.py b/test_data.py index 1434cce..205ab70 100644 --- a/test_data.py +++ b/test_data.py @@ -132,11 +132,11 @@ def print_values(self): class WaveSaver(object): - def __init__(self, n_trials, n_components): - self.amplitude = np.zeros((n_trials, n_components), dtype='float') - self.frequency = np.zeros((n_trials, n_components), dtype='float') - self.offset = np.zeros((n_trials, n_components), dtype='float') - self.damping = np.zeros((n_trials, n_components), dtype='float') + def __init__(self, n_trials, n_components, n_dim=1): + self.amplitude = np.zeros((n_trials, n_components, n_dim), dtype='float') + self.frequency = np.zeros((n_trials, n_components, n_dim), dtype='float') + self.offset = np.zeros((n_trials, n_components, n_dim), dtype='float') + self.damping = np.zeros((n_trials, n_components, n_dim), dtype='float') self.n = 0 def set_vals(self, n, a, f, o, d): From d5c68e5f855e468379dfb764c1f231750ac11024 Mon Sep 17 00:00:00 2001 From: Nicholas Lawrance Date: Sat, 23 Mar 2019 13:19:57 +0100 Subject: [PATCH 29/38] Tidy plotting routines --- GP_preference_demo.py | 6 +++--- ordinal_likelihoods.py | 2 +- plot_tools.py | 33 +++++++++++++++++++-------------- pref_active_learning.py | 14 +++++++------- 4 files changed, 30 insertions(+), 25 deletions(-) diff --git a/GP_preference_demo.py b/GP_preference_demo.py index 474d882..54bc60f 100644 --- a/GP_preference_demo.py +++ b/GP_preference_demo.py @@ -14,7 +14,7 @@ use_test_data = False # test_data.data3 # verbose = 2 -with open('./data/low_freq_2.yaml', 'rt') as fh: +with open('./data/statruns_2D.yaml', 'rt') as fh: wave = yaml.safe_load(fh) try: np.random.seed(wave['statrun_params']['randseed']) @@ -25,9 +25,9 @@ log_hyp = np.log(wave['hyperparameters']) n_rel_train = 5 -n_abs_train = 0 +n_abs_train = 5 -n_xplot = 101 +n_xplot = 21 n_posterior_samples = 3 random_wave.print_values() diff --git a/ordinal_likelihoods.py b/ordinal_likelihoods.py index deaceec..fe0dd9a 100644 --- a/ordinal_likelihoods.py +++ b/ordinal_likelihoods.py @@ -122,7 +122,7 @@ def fun(x): # ah4.set_xlabel('$y$') # ah4.set_ylabel('$p(y|f)$') # ah4.set_title('Symmetric beta likelihood') -plt.show() +plt.show(block=False) diff --git a/plot_tools.py b/plot_tools.py index 670fe7e..2c7d47d 100644 --- a/plot_tools.py +++ b/plot_tools.py @@ -45,26 +45,31 @@ def plot_with_bounds(ax, x, y, s, c=lines[0], lw=1.5, *args, **kwargs): ax.set_ylim(bottom = min(clim[0],xy[:,1].min()), top = max(clim[1], xy[:,1].max())) return h_fx, h_patch + +def _set_subplot_labels(ax, xlabel='$x$', ylabel='$f(x)$', title=''): + ax.set_title(title) + ax.set_xlabel(xlabel) + ax.set_ylabel(ylabel) + + +def plot_setup_1drel(t_l = r'Latent function, $f(x)$', t_r = r'Relative likelihood, $P(x_0 \succ x_1 | f(x_0), f(x_1))$'): + fig, (ax_l, ax_r) = plt.subplots(1, 2) + fig.set_size_inches(8.7, 3.2) + _set_subplot_labels(ax_l, title=t_l) # Latent function + _set_subplot_labels(ax_r, xlabel='$x_0$', ylabel='$x_1$', title=t_r) # Relative likelihood + return fig, (ax_l, ax_r) + + def plot_setup_1d(t_l = r'Latent function, $f(x)$', t_a = r'Absolute likelihood, $p(y | f(x))$', t_r = r'Relative likelihood, $P(x_0 \succ x_1 | f(x_0), f(x_1))$'): fig, (ax_l, ax_a, ax_r) = plt.subplots(1, 3) fig.set_size_inches(14.7, 3.5) - # Latent function - ax_l.set_title(t_l) - ax_l.set_xlabel('$x$') - ax_l.set_ylabel('$f(x)$') - - # Absolute likelihood - ax_a.set_title(t_a) - ax_a.set_xlabel('$x$') - ax_a.set_ylabel('$y$') + _set_subplot_labels(ax_l, title=t_l) # Latent function + _set_subplot_labels(ax_a, ylabel='$y$', title=t_a) # Absolute likelihood + _set_subplot_labels(ax_r, xlabel='$x_0$', ylabel='$x_1$', title=t_r) # Relative likelihood - # Relative likelihood - ax_r.set_title(t_r) - ax_r.set_xlabel('$x_0$') - ax_r.set_ylabel('$x_1$') return fig, (ax_l, ax_a, ax_r) @@ -142,7 +147,7 @@ def estimate_plots(xt, ft, mu_t, fhat, vhat, E_y, rel_sigma, ax_l.legend([hf, hf_hat], [r'True latent function, $f(x)$', r'$\mathcal{GP}$ estimate $\hat{f}(x)$']) # Absolute posterior likelihood - plot_absolute_likelihood(ax_a, p_a_y, xt, mu_t, y_samples, E_y=E_y, xa_train=xa_train, ya_train=xa_train) + plot_absolute_likelihood(ax_a, p_a_y, xt, mu_t, y_samples, E_y=E_y, xa_train=xa_train, ya_train=ya_train) # Relative posterior likelihood rel_y_extent = [xt[0, 0], xt[-1, 0], xt[0, 0], xt[-1, 0]] diff --git a/pref_active_learning.py b/pref_active_learning.py index 52c259f..0e17130 100644 --- a/pref_active_learning.py +++ b/pref_active_learning.py @@ -13,9 +13,9 @@ plt.rc('font',**{'family':'serif','sans-serif':['Computer Modern Roman']}) plt.rc('text', usetex=True) -save_plots = False +save_plots = True -with open('./data/statruns_nov2017.yaml', 'rt') as fh: +with open('./data/params.yaml', 'rt') as fh: wave = yaml.safe_load(fh) try: @@ -61,15 +61,15 @@ # Training data - note the shifted domain to get the sample out of the way far_domain = np.array([[-3.0], [-2.0]]) -x_rel, uvi_rel, uv_rel, y_rel, fuv_rel = rel_obs_fun.generate_n_observations(n_rel_train, n_xdim=1, domain=far_domain) +x_rel, uvi_rel, y_rel, fuv_rel = rel_obs_fun.generate_n_observations(n_rel_train, n_xdim=1, domain=far_domain) x_abs, y_abs, mu_abs = abs_obs_fun.generate_n_observations(n_abs_train, n_xdim=1, domain=far_domain) # Plot true functions -fig_t, (ax_t_l, ax_t_a, ax_t_r) = ptt.true_plots(x_test, f_true, mu_true, rel_sigma, - abs_y_samples, p_abs_y_true, p_rel_y_true, - x_abs, y_abs, uv_rel, fuv_rel, y_rel, - t_l=r'True latent function, $f(x)$') +fig_t, (ax_t_l, ax_t_a, ax_t_r) = ptt.true_plots(x_test, f_true, mu_true, wave['rel_obs_params']['sigma'], + abs_y_samples, p_abs_y_true, p_rel_y_true, + t_l=r'True latent function, $f(x)$') + if save_plots: fig_t.savefig(fig_dir+'true.pdf', bbox_inches='tight') From 7f3d49eac56f1c11ad0559a0a396ef3a116aa508 Mon Sep 17 00:00:00 2001 From: Nicholas Lawrance Date: Sun, 24 Mar 2019 16:12:13 +0100 Subject: [PATCH 30/38] Add wine data --- data/wine_quality/winequality.names | 72 +++++++++++++++++++++++++++++ read_wine_data.py | 24 ++++++++++ 2 files changed, 96 insertions(+) create mode 100644 data/wine_quality/winequality.names create mode 100644 read_wine_data.py diff --git a/data/wine_quality/winequality.names b/data/wine_quality/winequality.names new file mode 100644 index 0000000..4e1de1f --- /dev/null +++ b/data/wine_quality/winequality.names @@ -0,0 +1,72 @@ +Citation Request: + This dataset is public available for research. The details are described in [Cortez et al., 2009]. + Please include this citation if you plan to use this database: + + P. Cortez, A. Cerdeira, F. Almeida, T. Matos and J. Reis. + Modeling wine preferences by data mining from physicochemical properties. + In Decision Support Systems, Elsevier, 47(4):547-553. ISSN: 0167-9236. + + Available at: [@Elsevier] http://dx.doi.org/10.1016/j.dss.2009.05.016 + [Pre-press (pdf)] http://www3.dsi.uminho.pt/pcortez/winequality09.pdf + [bib] http://www3.dsi.uminho.pt/pcortez/dss09.bib + +1. Title: Wine Quality + +2. Sources + Created by: Paulo Cortez (Univ. Minho), Antonio Cerdeira, Fernando Almeida, Telmo Matos and Jose Reis (CVRVV) @ 2009 + +3. Past Usage: + + P. Cortez, A. Cerdeira, F. Almeida, T. Matos and J. Reis. + Modeling wine preferences by data mining from physicochemical properties. + In Decision Support Systems, Elsevier, 47(4):547-553. ISSN: 0167-9236. + + In the above reference, two datasets were created, using red and white wine samples. + The inputs include objective tests (e.g. PH values) and the output is based on sensory data + (median of at least 3 evaluations made by wine experts). Each expert graded the wine quality + between 0 (very bad) and 10 (very excellent). Several data mining methods were applied to model + these datasets under a regression approach. The support vector machine model achieved the + best results. Several metrics were computed: MAD, confusion matrix for a fixed error tolerance (T), + etc. Also, we plot the relative importances of the input variables (as measured by a sensitivity + analysis procedure). + +4. Relevant Information: + + The two datasets are related to red and white variants of the Portuguese "Vinho Verde" wine. + For more details, consult: http://www.vinhoverde.pt/en/ or the reference [Cortez et al., 2009]. + Due to privacy and logistic issues, only physicochemical (inputs) and sensory (the output) variables + are available (e.g. there is no data about grape types, wine brand, wine selling price, etc.). + + These datasets can be viewed as classification or regression tasks. + The classes are ordered and not balanced (e.g. there are munch more normal wines than + excellent or poor ones). Outlier detection algorithms could be used to detect the few excellent + or poor wines. Also, we are not sure if all input variables are relevant. So + it could be interesting to test feature selection methods. + +5. Number of Instances: red wine - 1599; white wine - 4898. + +6. Number of Attributes: 11 + output attribute + + Note: several of the attributes may be correlated, thus it makes sense to apply some sort of + feature selection. + +7. Attribute information: + + For more information, read [Cortez et al., 2009]. + + Input variables (based on physicochemical tests): + 1 - fixed acidity + 2 - volatile acidity + 3 - citric acid + 4 - residual sugar + 5 - chlorides + 6 - free sulfur dioxide + 7 - total sulfur dioxide + 8 - density + 9 - pH + 10 - sulphates + 11 - alcohol + Output variable (based on sensory data): + 12 - quality (score between 0 and 10) + +8. Missing Attribute Values: None diff --git a/read_wine_data.py b/read_wine_data.py new file mode 100644 index 0000000..ee5924a --- /dev/null +++ b/read_wine_data.py @@ -0,0 +1,24 @@ +import numpy as np +import pandas +import matplotlib.pyplot as plt + +class WineQualityData(object): + + def __init__(self, data_file): + self.file = data_file + self.data = pandas.read_csv(self.file, delimiter=';') + +red_data = WineQualityData('data/wine_quality/winequality-red.csv') +white_data = WineQualityData('data/wine_quality/winequality-white.csv') + +fh, ah = plt.subplots(1, 2) +ah[0].hist(red_data.data.quality, np.arange(-0.5, 11, 1.0)) +ah[1].hist(white_data.data.quality, np.arange(-0.5, 11, 1.0)) +for a in ah: + a.set_xticks(np.arange(11)) + a.set_xlabel('Score') +ah[0].set_ylabel('Count') +ah[0].set_title('Red wine') +ah[1].set_title('White wine') + +plt.show(block=False) \ No newline at end of file From e4a58de1ca4f32a0c22cc160477621034e188731 Mon Sep 17 00:00:00 2001 From: Nicholas Lawrance Date: Sun, 24 Mar 2019 16:27:12 +0100 Subject: [PATCH 31/38] Add wine data wget commands --- read_wine_data.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/read_wine_data.py b/read_wine_data.py index ee5924a..e942cbd 100644 --- a/read_wine_data.py +++ b/read_wine_data.py @@ -2,6 +2,9 @@ import pandas import matplotlib.pyplot as plt +# Get wine data: +# wget -P data/wine_quality https://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-red.csv +# wget -P data/wine_quality https://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-white.csv class WineQualityData(object): def __init__(self, data_file): From 66f278873a11079951ca36a1d70c8838941f90bc Mon Sep 17 00:00:00 2001 From: Nicholas Lawrance Date: Thu, 28 Mar 2019 00:19:39 +0100 Subject: [PATCH 32/38] Add wine data basic tests --- GP_preference_demo.py | 32 ++++------- {data => config}/statruns2_oct2017.yaml | 0 data/wine_quality/winequality.names | 72 ------------------------- plot_tools.py | 8 ++- read_wine_data.py | 29 +++++++++- 5 files changed, 45 insertions(+), 96 deletions(-) rename {data => config}/statruns2_oct2017.yaml (100%) delete mode 100644 data/wine_quality/winequality.names diff --git a/GP_preference_demo.py b/GP_preference_demo.py index 1503c3e..a45f729 100644 --- a/GP_preference_demo.py +++ b/GP_preference_demo.py @@ -27,7 +27,7 @@ n_rel_train = 5 n_abs_train = 5 -n_xplot = 101 +n_xplot = 31 n_posterior_samples = 3 random_wave.print_values() @@ -89,38 +89,26 @@ if d_x <= 2: # Plot true functions - fig_t, (ax_t_l, ax_t_a, ax_t_r) = ptt.true_plots(x_test, f_true, mu_true, wave['rel_obs_params']['sigma'], + fig_t, ax_t = ptt.true_plots(x_test, f_true, mu_true, wave['rel_obs_params']['sigma'], abs_y_samples, p_abs_y_true, p_rel_y_true, t_l=r'True latent function, $f(x)$') # Posterior estimates - fig_p, (ax_p_l, ax_p_a, ax_p_r) = \ + fig_p, ax_p = \ ptt.estimate_plots(x_test, f_true, mu_true, fhat, vhat, E_y, wave['rel_obs_params']['sigma'], abs_y_samples, p_abs_y_post, p_rel_y_post, - x_abs, y_abs, x_rel[uvi_rel][:,:,0], fuv_rel, y_rel, n_posterior_samples=n_posterior_samples, + x_abs, y_abs, x_rel[uvi_rel], fuv_rel, y_rel, n_posterior_samples=n_posterior_samples, posterior_plot_kwargs={'color':'grey', 'ls':'--'}, + t_l=r'$\mathcal{GP}$ latent function estimate $\hat{f}(x)$', t_a=r'Posterior absolute likelihood, $p(u | \mathcal{Y}, \theta)$', t_r=r'Posterior relative likelihood $P(x_0 \succ x_1 | \mathcal{Y}, \theta)$') - p_err = test_data.rel_error(mu_true, p_rel_y_true, E_y, p_rel_y_post, weight=False) - print "WRMS: {0:0.3f}, WRMS_MC: {1:0.3f}, p_err: {2:0.3f}".format(wrms, wrms2, p_err) + if d_x == 1: + p_err = test_data.rel_error(mu_true, p_rel_y_true, E_y, p_rel_y_post, weight=False) + print "WRMS: {0:0.3f}, WRMS_MC: {1:0.3f}, p_err: {2:0.3f}".format(wrms, wrms2, p_err) + else: + print "WRMS: {0:0.3f}, WRMS_MC: {1:0.3f}".format(wrms, wrms2) plt.show(block=False) - -elif d_x is 2: - # Plot true functions - fig_t, (ax_t_l, ax_t_a) = ptt.true_plots2D(x_test, f_true, mu_true, wave['rel_obs_params']['sigma'], - abs_y_samples, p_abs_y_true, t_l=r'True latent function, $f(x)$') - - # Posterior estimates - fig_p, (ax_p_l, ax_p_a) = \ - ptt.estimate_plots2D(x_test, f_true, mu_true, fhat, vhat, E_y, wave['rel_obs_params']['sigma'], - abs_y_samples, p_abs_y_post, x_abs, y_abs, x_rel[uvi_rel], fuv_rel, y_rel, - t_l=r'$\mathcal{GP}$ latent function estimate $\hat{f}(x)$', - t_a=r'Posterior absolute likelihood, $p(u | \mathcal{Y}, \theta)$') - print "WRMS: {0:0.3f}, WRMS_MC: {1:0.3f}".format(wrms, wrms2) - plt.show(block=False) - - else: print "Input state space dimension: {0} is too high to plot".format(d_x) print "WRMS: {0:0.3f}, WRMS_MC: {1:0.3f}".format(wrms, wrms2) diff --git a/data/statruns2_oct2017.yaml b/config/statruns2_oct2017.yaml similarity index 100% rename from data/statruns2_oct2017.yaml rename to config/statruns2_oct2017.yaml diff --git a/data/wine_quality/winequality.names b/data/wine_quality/winequality.names deleted file mode 100644 index 4e1de1f..0000000 --- a/data/wine_quality/winequality.names +++ /dev/null @@ -1,72 +0,0 @@ -Citation Request: - This dataset is public available for research. The details are described in [Cortez et al., 2009]. - Please include this citation if you plan to use this database: - - P. Cortez, A. Cerdeira, F. Almeida, T. Matos and J. Reis. - Modeling wine preferences by data mining from physicochemical properties. - In Decision Support Systems, Elsevier, 47(4):547-553. ISSN: 0167-9236. - - Available at: [@Elsevier] http://dx.doi.org/10.1016/j.dss.2009.05.016 - [Pre-press (pdf)] http://www3.dsi.uminho.pt/pcortez/winequality09.pdf - [bib] http://www3.dsi.uminho.pt/pcortez/dss09.bib - -1. Title: Wine Quality - -2. Sources - Created by: Paulo Cortez (Univ. Minho), Antonio Cerdeira, Fernando Almeida, Telmo Matos and Jose Reis (CVRVV) @ 2009 - -3. Past Usage: - - P. Cortez, A. Cerdeira, F. Almeida, T. Matos and J. Reis. - Modeling wine preferences by data mining from physicochemical properties. - In Decision Support Systems, Elsevier, 47(4):547-553. ISSN: 0167-9236. - - In the above reference, two datasets were created, using red and white wine samples. - The inputs include objective tests (e.g. PH values) and the output is based on sensory data - (median of at least 3 evaluations made by wine experts). Each expert graded the wine quality - between 0 (very bad) and 10 (very excellent). Several data mining methods were applied to model - these datasets under a regression approach. The support vector machine model achieved the - best results. Several metrics were computed: MAD, confusion matrix for a fixed error tolerance (T), - etc. Also, we plot the relative importances of the input variables (as measured by a sensitivity - analysis procedure). - -4. Relevant Information: - - The two datasets are related to red and white variants of the Portuguese "Vinho Verde" wine. - For more details, consult: http://www.vinhoverde.pt/en/ or the reference [Cortez et al., 2009]. - Due to privacy and logistic issues, only physicochemical (inputs) and sensory (the output) variables - are available (e.g. there is no data about grape types, wine brand, wine selling price, etc.). - - These datasets can be viewed as classification or regression tasks. - The classes are ordered and not balanced (e.g. there are munch more normal wines than - excellent or poor ones). Outlier detection algorithms could be used to detect the few excellent - or poor wines. Also, we are not sure if all input variables are relevant. So - it could be interesting to test feature selection methods. - -5. Number of Instances: red wine - 1599; white wine - 4898. - -6. Number of Attributes: 11 + output attribute - - Note: several of the attributes may be correlated, thus it makes sense to apply some sort of - feature selection. - -7. Attribute information: - - For more information, read [Cortez et al., 2009]. - - Input variables (based on physicochemical tests): - 1 - fixed acidity - 2 - volatile acidity - 3 - citric acid - 4 - residual sugar - 5 - chlorides - 6 - free sulfur dioxide - 7 - total sulfur dioxide - 8 - density - 9 - pH - 10 - sulphates - 11 - alcohol - Output variable (based on sensory data): - 12 - quality (score between 0 and 10) - -8. Missing Attribute Values: None diff --git a/plot_tools.py b/plot_tools.py index 446119d..24b2b25 100644 --- a/plot_tools.py +++ b/plot_tools.py @@ -3,6 +3,7 @@ import matplotlib.pyplot as plt from matplotlib.patches import Polygon import matplotlib.cm as cm +import matplotlib.colors from nice_plot_colors import * from cycler import cycler from mpl_toolkits.mplot3d.art3d import Line3DCollection @@ -232,7 +233,12 @@ def estimate_plots2D(xt, ft, mu_t, fhat, vhat, E_y, rel_sigma, # Latent function estimate # hf =ax_l.plot_wireframe(xx, yy, np.reshape(ft, (nx, nx)), color=lines[0]) tfc = np.reshape(vhat.diagonal(), (nx, nx)) - hf_hat = ax_l.plot_surface(xx, yy, np.reshape(fhat, (nx, nx)), facecolors=cc(tfc/tfc.max())) + norm = matplotlib.colors.Normalize(vmin=0, vmax=tfc.max()) + hf_hat = ax_l.plot_surface(xx, yy, np.reshape(fhat, (nx, nx)), facecolors=cc(norm(tfc))) + m = cm.ScalarMappable(cmap=cc, norm=norm) + m.set_array([]) + fig.colorbar(m, ax=ax_l) + # ax_l.imshow(np.reshape(fhat, (nx, nx)), extent=[x0[0], x1[-1], x1[0], x1[1]], origin='lower', aspect='auto') # ax_l.legend([hf], [r'True latent function, $f(x)$']) #, r'$\mathcal{GP}$ estimate $\hat{f}(x)$']) diff --git a/read_wine_data.py b/read_wine_data.py index e942cbd..ba6283b 100644 --- a/read_wine_data.py +++ b/read_wine_data.py @@ -1,6 +1,9 @@ import numpy as np import pandas import matplotlib.pyplot as plt +import yaml +import GPpref +import scipy.optimize as op # Get wine data: # wget -P data/wine_quality https://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-red.csv @@ -24,4 +27,28 @@ def __init__(self, data_file): ah[0].set_title('Red wine') ah[1].set_title('White wine') -plt.show(block=False) \ No newline at end of file +plt.show(block=False) + +with open('./data/statruns_wine.yaml', 'rt') as fh: + config = yaml.safe_load(fh) +d_x = config['GP_params']['hyper_counts'][0]-1 +log_hyp = np.log(config['hyperparameters']) + +n_rel = 1 +n_abs = 100 +x_rel = red_data.data.values[0:2*n_rel, 0:-1] +uvi_rel = np.array([[0, 1]]) +fuv_rel = red_data.data.values[0:2*n_rel, -1] +y_rel = np.array([1]) + +x_abs = red_data.data.values[2*n_rel:(2*n_rel+n_abs), 0:-1] +y_abs = np.expand_dims(red_data.data.values[2*n_rel:(2*n_rel+n_abs), -1]-2, axis=1).astype(dtype='int') +model_kwargs = {'x_rel': x_rel, 'uvi_rel': uvi_rel, 'x_abs': x_abs, 'y_rel': y_rel, 'y_abs': y_abs, + 'rel_kwargs': config['rel_obs_params'], 'abs_kwargs': config['abs_obs_params']} +model_kwargs.update(config['GP_params']) +model_kwargs['verbose'] = 2 +prefGP = GPpref.PreferenceGaussianProcess(**model_kwargs) + +prefGP.set_hyperparameters(log_hyp) +f = prefGP.calc_laplace(log_hyp) +log_hyp_opt = op.fmin(prefGP.calc_nlml,log_hyp) \ No newline at end of file From a0515bdcc0e0a3aa1033b05e52761b286e9854d1 Mon Sep 17 00:00:00 2001 From: Nicholas Lawrance Date: Sat, 25 May 2019 14:18:21 +0200 Subject: [PATCH 33/38] Add video capability to pref_active_learning - Currently a bit hacky (make new image for each frame, expensive but works in 2D, 1D not tested --- GP_preference_demo.py | 2 +- active_learners.py | 26 +++++++++++++++- plot_acquisition.py | 6 ++-- plot_tools.py | 63 +++++++++++++++++++++++++++------------ pref_active_learning.py | 66 +++++++++++++++++++++++++++++++++-------- read_wine_data.py | 6 ++-- 6 files changed, 130 insertions(+), 39 deletions(-) diff --git a/GP_preference_demo.py b/GP_preference_demo.py index a45f729..49c8933 100644 --- a/GP_preference_demo.py +++ b/GP_preference_demo.py @@ -94,7 +94,7 @@ t_l=r'True latent function, $f(x)$') # Posterior estimates - fig_p, ax_p = \ + fig_p, ax_p, h_plotobj = \ ptt.estimate_plots(x_test, f_true, mu_true, fhat, vhat, E_y, wave['rel_obs_params']['sigma'], abs_y_samples, p_abs_y_post, p_rel_y_post, x_abs, y_abs, x_rel[uvi_rel], fuv_rel, y_rel, n_posterior_samples=n_posterior_samples, diff --git a/active_learners.py b/active_learners.py index 598dcc6..cec0103 100644 --- a/active_learners.py +++ b/active_learners.py @@ -64,7 +64,8 @@ def linear_domain_sampler(self, n_samples, domain=None): x_test = x_test*np.diff(domain, axis=0) + domain[0, :] return x_test - def create_posterior_plot(self, x_test, f_true, mu_true, rel_sigma, fuv_train, abs_y_samples, **plot_kwargs): + + def get_latent_ests(self, x_test, abs_y_samples ): # Latent predictions d_x = x_test.shape[1] fhat, vhat = self.predict_latent(x_test) @@ -73,6 +74,13 @@ def create_posterior_plot(self, x_test, f_true, mu_true, rel_sigma, fuv_train, a p_rel_y_post = self.rel_posterior_likelihood_array(fhat=fhat, varhat=vhat) else: p_rel_y_post = None + return fhat, vhat, p_abs_y_post, E_y, p_rel_y_post + + + def create_posterior_plot(self, x_test, f_true, mu_true, rel_sigma, fuv_train, abs_y_samples, **plot_kwargs): + + fhat, vhat, p_abs_y_post, E_y, p_rel_y_post = self.get_latent_ests(x_test, abs_y_samples) + x_train, uvi_train, x_abs_train, y_train, y_abs_train = self.get_observations() uv_train = x_train[uvi_train] @@ -84,6 +92,22 @@ def create_posterior_plot(self, x_test, f_true, mu_true, rel_sigma, fuv_train, a t_l=r'$\mathcal{GP}$ latent function estimate $\hat{f}(x)$', t_a=r'Posterior absolute likelihood, $p(y | \mathcal{Y}, \theta)$', t_r=r'Posterior relative likelihood $P(x_0 \succ x_1 | \mathcal{Y}, \theta)$', **plot_kwargs) + return fig_p, ax_p, [fhat, vhat, p_abs_y_post, E_y, p_rel_y_post] + + def update_posterior_plot(self, x_test, f_true, mu_true, rel_sigma, fuv_train, abs_y_samples, + fhat, vhat, p_abs_y_post, E_y, p_rel_y_post, ax_p): + x_train, uvi_train, x_abs_train, y_train, y_abs_train = self.get_observations() + uv_train = x_train[uvi_train] + + ptt.reset_axes2d(ax_p) + # Posterior estimates + fig_p, ax_p = \ + ptt.estimate_plots(x_test, f_true, mu_true, fhat, vhat, E_y, rel_sigma, + abs_y_samples, p_abs_y_post, p_rel_y_post, + x_abs_train, y_abs_train, uv_train, fuv_train, y_train, + t_l=r'$\mathcal{GP}$ latent function estimate $\hat{f}(x)$', + t_a=r'Posterior absolute likelihood, $p(y | \mathcal{Y}, \theta)$', + t_r=r'Posterior relative likelihood $P(x_0 \succ x_1 | \mathcal{Y}, \theta)$', ax=ax_p) return fig_p, ax_p diff --git a/plot_acquisition.py b/plot_acquisition.py index e828957..26f97c1 100644 --- a/plot_acquisition.py +++ b/plot_acquisition.py @@ -95,7 +95,7 @@ learner.print_hyperparameters() if save_plots: - fig_p, (ax_p_l, ax_p_a, ax_p_r) = learner.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, + fig_p, (ax_p_l, ax_p_a, ax_p_r), h_plotobj = learner.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, abs_y_samples, mc_samples) fig_a, ax_a = plt.subplots() pdf_pages.savefig(fig_p, bbox_inches='tight') @@ -118,7 +118,7 @@ f = learner.solve_laplace() if save_plots: - fig_p, (ax_p_l, ax_p_a, ax_p_r) = learner.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, + fig_p, (ax_p_l, ax_p_a, ax_p_r), h_plotobj = learner.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, abs_y_samples, mc_samples) pdf_pages.savefig(fig_p, bbox_inches='tight') # fig_p.savefig(fig_dir+'posterior{0:02d}.pdf'.format(obs_num+1), bbox_inches='tight') @@ -127,7 +127,7 @@ learner.print_hyperparameters() if not save_plots: - fig_p, (ax_p_l, ax_p_a, ax_p_r) = learner.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, + fig_p, (ax_p_l, ax_p_a, ax_p_r), h_plotobj = learner.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, abs_y_samples, mc_samples) else: pdf_pages.close() diff --git a/plot_tools.py b/plot_tools.py index 24b2b25..faebfee 100644 --- a/plot_tools.py +++ b/plot_tools.py @@ -53,8 +53,12 @@ def _set_subplot_labels(ax, xlabel='$x$', ylabel='$f(x)$', title=''): ax.set_ylabel(ylabel) -def plot_setup_1drel(t_l = r'Latent function, $f(x)$', t_r = r'Relative likelihood, $P(x_0 \succ x_1 | f(x_0), f(x_1))$'): - fig, (ax_l, ax_r) = plt.subplots(1, 2) +def plot_setup_1drel(t_l = r'Latent function, $f(x)$', t_r = r'Relative likelihood, $P(x_0 \succ x_1 | f(x_0), f(x_1))$', ax=None, **kwargs): + if ax is None: + fig, (ax_l, ax_r) = plt.subplots(1, 2, **kwargs) + else: + (ax_l, ax_r) = ax + fig = ax_l.figure fig.set_size_inches(8.7, 3.2) _set_subplot_labels(ax_l, title=t_l) # Latent function _set_subplot_labels(ax_r, xlabel='$x_0$', ylabel='$x_1$', title=t_r) # Relative likelihood @@ -62,10 +66,17 @@ def plot_setup_1drel(t_l = r'Latent function, $f(x)$', t_r = r'Relative likeliho def plot_setup_1d(t_l = r'Latent function, $f(x)$', t_a = r'Absolute likelihood, $p(y | f(x))$', - t_r = r'Relative likelihood, $P(x_0 \succ x_1 | f(x_0), f(x_1))$', **kwargs): - - fig, (ax_l, ax_a, ax_r) = plt.subplots(1, 3, **kwargs) - fig.set_size_inches(14.7, 3.5) + t_r = r'Relative likelihood, $P(x_0 \succ x_1 | f(x_0), f(x_1))$', ax = None, **kwargs): + + if ax is None: + fig = plt.figure() + ax_l = fig.add_subplot(131, **kwargs) + ax_a = fig.add_subplot(132, **kwargs) + ax_r = fig.add_subplot(133, **kwargs) + fig.set_size_inches(14.7, 3.5) + else: + (ax_l, ax_a, ax_r) = ax + fig = ax_l.figure _set_subplot_labels(ax_l, title=t_l) # Latent function _set_subplot_labels(ax_a, ylabel='$y$', title=t_a) # Absolute likelihood @@ -76,7 +87,7 @@ def plot_setup_1d(t_l = r'Latent function, $f(x)$', t_a = r'Absolute likelihood, def plot_relative_queries(ax, uvr_train, fuvr_train, yr_train, class_icons, marker_options=None): for uv, fuv, y in zip(uvr_train, fuvr_train, yr_train): - ax.plot(uv, fuv, 'b-', color=lighten(lines[0])) + ax.plot(uv, fuv, 'b-', color=lighten(lines[0]), lw=0.8) ax.plot(uv[(y + 1) / 2], fuv[(y + 1) / 2], class_icons[(y[0] + 1) / 2], **marker_options) @@ -95,7 +106,7 @@ def plot_absolute_likelihood(ax, p_a_y, xt, mu_t, y_samples, E_y=None, xa_train= ax.legend(h_lines, legend_entries) ax.set_xlim(xt[0], xt[-1]) ax.get_figure().colorbar(h_pat, ax=ax) - return h_pat + return (h_pat, h_lines) def plot_relative_likelihood(ax, p_y, extent, uvr_train=None, yr_train=None, class_icons=None, marker_options=None): h_p = ax.imshow(p_y, origin='lower', extent=extent, vmin=0.0, vmax=1.0) @@ -106,7 +117,7 @@ def plot_relative_likelihood(ax, p_y, extent, uvr_train=None, yr_train=None, cla if uvr_train is not None: # and xt.shape[0] > 0: for uv, y in zip(uvr_train, yr_train): ax.plot(uv[0], uv[1], class_icons[(y[0] + 1) / 2], **marker_options) - return h_p + return (h_p, h_pc) def true_plots(xt, ft, mu_t, rel_sigma, y_samples, p_a_y, p_r_y=None, xa_train=None, ya_train=None, uvr_train=None, fuvr_train=None, yr_train=None, class_icons=['ko', 'wo'], marker_options={'mec':'k', 'mew':0.5}, *args, **kwargs): @@ -130,6 +141,7 @@ def true_plots(xt, ft, mu_t, rel_sigma, y_samples, p_a_y, p_r_y=None, xa_train=N h_prt = plot_relative_likelihood(ax_r, p_r_y, rel_y_extent, uvr_train, yr_train, class_icons, marker_options) return fig, (ax_l, ax_a, ax_r) + def estimate_plots(xt, ft, mu_t, fhat, vhat, E_y, rel_sigma, y_samples, p_a_y, p_r_y, xa_train, ya_train, uvr_train, fuvr_train, yr_train, class_icons = ['ko', 'wo'], marker_options = {'mec':'k', 'mew':0.5}, n_posterior_samples=0, @@ -145,6 +157,8 @@ def estimate_plots(xt, ft, mu_t, fhat, vhat, E_y, rel_sigma, if n_posterior_samples > 0: f_post = np.random.multivariate_normal(fhat.flatten(), vhat, n_posterior_samples) h_pp = ax_l.plot(xt, f_post.T, lw=0.8, **posterior_plot_kwargs) + else: + h_pp = None # Latent function hf, hpf = plot_with_bounds(ax_l, xt, ft, rel_sigma, c=lines[0]) @@ -155,22 +169,27 @@ def estimate_plots(xt, ft, mu_t, fhat, vhat, E_y, rel_sigma, ax_l.legend([hf, hf_hat], [r'True latent function, $f(x)$', r'$\mathcal{GP}$ estimate $\hat{f}(x)$']) # Absolute posterior likelihood - plot_absolute_likelihood(ax_a, p_a_y, xt, mu_t, y_samples, E_y=E_y, xa_train=xa_train, ya_train=ya_train) + h_abs = plot_absolute_likelihood(ax_a, p_a_y, xt, mu_t, y_samples, E_y=E_y, xa_train=xa_train, ya_train=ya_train) # Relative posterior likelihood rel_y_extent = [xt[0, 0], xt[-1, 0], xt[0, 0], xt[-1, 0]] - h_prp = plot_relative_likelihood(ax_r, p_r_y, rel_y_extent, uvr_train, yr_train, class_icons, marker_options) + h_rel = plot_relative_likelihood(ax_r, p_r_y, rel_y_extent, uvr_train, yr_train, class_icons, marker_options) return fig, (ax_l, ax_a, ax_r) + # 2D Plot tools -def plot_setup_2d(t_l = r'Latent function, $f(x)$', t_a = r'Absolute likelihood, $p(y | f(x))$', t_r = None, **kwargs): +def plot_setup_2d(t_l = r'Latent function, $f(x)$', t_a = r'Absolute likelihood, $p(y | f(x))$', t_r = None, ax = None, **kwargs): - fig = plt.figure() - ax_l = fig.add_subplot(121, projection='3d', **kwargs) - ax_a = fig.add_subplot(122, projection='3d', **kwargs) - fig.set_size_inches(10.0, 3.5) + if ax is None: + fig = plt.figure() + ax_l = fig.add_subplot(121, projection='3d', **kwargs) + ax_a = fig.add_subplot(122, projection='3d', **kwargs) + fig.set_size_inches(10.0, 3.5) + else: + (ax_l, ax_a) = ax + fig = ax_l.figure # Latent function ax_l.set_title(t_l) @@ -237,7 +256,7 @@ def estimate_plots2D(xt, ft, mu_t, fhat, vhat, E_y, rel_sigma, hf_hat = ax_l.plot_surface(xx, yy, np.reshape(fhat, (nx, nx)), facecolors=cc(norm(tfc))) m = cm.ScalarMappable(cmap=cc, norm=norm) m.set_array([]) - fig.colorbar(m, ax=ax_l) + # ax_cb = fig.colorbar(m, ax=ax_l) # ax_l.imshow(np.reshape(fhat, (nx, nx)), extent=[x0[0], x1[-1], x1[0], x1[1]], origin='lower', aspect='auto') # ax_l.legend([hf], [r'True latent function, $f(x)$']) #, r'$\mathcal{GP}$ estimate $\hat{f}(x)$']) @@ -258,18 +277,24 @@ def estimate_plots2D(xt, ft, mu_t, fhat, vhat, E_y, rel_sigma, cc = cm.get_cmap('Blues') norm_py = p_a_y/p_a_y.max() + h_points = [] for y, py in zip(y_samples, norm_py): - ax_a.scatter(xt[:, 0], xt[:, 1], y, s=py*15.0, marker='o', c=cc(py)) + h_points.append(ax_a.scatter(xt[:, 0], xt[:, 1], y, s=py*15.0, marker='o', c=cc(py))) if xa_train.shape[0] > 0: ax_a.plot(xa_train[:,0], xa_train[:,1], ya_train.flat, 'r^', color=lines[1]) ax_a.legend([hEy], [r'Posterior mean, $E_{p(y|\mathcal{Y})}\left[y\right]$']) # fig.colorbar(h_pap, ax=ax_a) - return fig, (ax_l, ax_a) +def reset_axes2d(est_ax): + for ax in est_ax: + ax.cla() + ax.set_xlim([0.0, 1.0]) + ax.set_ylim([0.0, 1.0]) + def ensure_dir(file_path): directory = os.path.dirname(file_path) if not os.path.exists(directory): diff --git a/pref_active_learning.py b/pref_active_learning.py index 057a44e..17bad23 100644 --- a/pref_active_learning.py +++ b/pref_active_learning.py @@ -13,7 +13,9 @@ plt.rc('font',**{'family':'serif','sans-serif':['Computer Modern Roman']}) plt.rc('text', usetex=True) -save_plots = False +save_plots = True +plot_type = 'video_frames' # 'pdf' +inter_frames = 10 with open('./data/shortrun_2D.yaml', 'rt') as fh: wave = yaml.safe_load(fh) @@ -30,7 +32,7 @@ n_xplot = 31 keep_f = True -learner_index = -1 +learner_index = -2 log_hyp = np.log(wave['hyperparameters']) @@ -45,7 +47,9 @@ fig_dir = 'fig/' + nowstr + '/' ptt.ensure_dir(fig_dir) print "Figures will be saved to: {0}".format(fig_dir) - pdf_pages = PdfPages(fig_dir+'posterior_all.pdf') + if plot_type == 'pdf': + pdf_pages = PdfPages(fig_dir+'posterior_all.pdf') + rel_obs_fun = GPpref.RelObservationSampler(true_function, wave['GP_params']['rel_likelihood'], wave['rel_obs_params']) abs_obs_fun = GPpref.AbsObservationSampler(true_function, wave['GP_params']['abs_likelihood'], wave['abs_obs_params']) @@ -71,11 +75,28 @@ # Plot true functions plot_kwargs = {'xlim':[0.0, 1.0], 'ylim':[0.0, 1.0]} +if plot_type != 'pdf': + if p_rel_y_true is None: + ax_t = [[], []] + fig_t = plt.figure() + fig_t.set_size_inches([11.44, 8.02]) + ax_t[0].append(fig_t.add_subplot(221, projection='3d', **plot_kwargs)) + ax_t[0].append(fig_t.add_subplot(222, projection='3d', **plot_kwargs)) + ax_t[1].append(fig_t.add_subplot(223, projection='3d', **plot_kwargs)) + ax_t[1].append(fig_t.add_subplot(224, projection='3d', **plot_kwargs)) + else: + fig_t, ax_t = plt.subplots(2,3) + true_ax = ax_t[0] + est_ax = ax_t[1] +else: + true_ax = None + est_ax = None + fig_t, ax_t = ptt.true_plots(x_test, f_true, mu_true, rel_sigma, abs_y_samples, p_abs_y_true, p_rel_y_true, x_abs, y_abs, x_rel[uvi_rel], fuv_rel, y_rel, - t_l=r'True latent function, $f(x)$', **plot_kwargs) -if save_plots: + t_l=r'True latent function, $f(x)$', ax=true_ax, **plot_kwargs) +if save_plots and (plot_type is 'pdf'): fig_t.savefig(fig_dir+'true.pdf', bbox_inches='tight') # Construct active learner object @@ -95,13 +116,15 @@ f = learner.model.solve_laplace() learner.model.print_hyperparameters() -fig_p, ax_p = learner.model.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, abs_y_samples, **plot_kwargs) +fig_p, ax_p, pre_data = learner.model.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, abs_y_samples, ax=est_ax, **plot_kwargs) if save_plots: wave['learners'] = [learner_kwargs] with open(fig_dir+'params.yaml', 'wt') as fh: yaml.safe_dump(wave, fh) - pdf_pages.savefig(fig_p, bbox_inches='tight') - # fig_p.savefig(fig_dir+'posterior00.pdf', bbox_inches='tight') + if plot_type is 'pdf': + pdf_pages.savefig(fig_p, bbox_inches='tight') + else: + fig_p.savefig(fig_dir+'posterior000.png', bbox_inches='tight') print learner_kwargs for obs_num in range(n_queries): @@ -126,17 +149,34 @@ f = learner.model.solve_laplace() if save_plots: - fig_p, ax_p = learner.model.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, abs_y_samples, **plot_kwargs) - pdf_pages.savefig(fig_p, bbox_inches='tight') - # fig_p.savefig(fig_dir+'posterior{0:02d}.pdf'.format(obs_num+1), bbox_inches='tight') + ptt.reset_axes2d(est_ax) + fig_p, ax_p, post_data = learner.model.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, + abs_y_samples, ax=est_ax, **plot_kwargs) + if plot_type is 'pdf': + pdf_pages.savefig(fig_p, bbox_inches='tight') + else: + fig_p.savefig(fig_dir + 'posterior{0:03d}.png'.format((obs_num+1)*inter_frames), bbox_inches='tight') + for iframe in range(1, inter_frames): + scale = float(iframe)/inter_frames + if pre_data[-1] is None: + pre_data = pre_data[:-1] + inter_data = [(1-scale)*pre + scale*post for pre, post in zip(pre_data, post_data)] + if len(inter_data) < len(post_data): + inter_data.append(None) + fig_p, ax_p = learner.model.update_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, + abs_y_samples, *inter_data, ax_p=est_ax) + fig_p.savefig(fig_dir + 'posterior{0:03d}.png'.format(obs_num*inter_frames + iframe), bbox_inches='tight') + pre_data = post_data plt.close(fig_p) + # fig_p.savefig(fig_dir+'posterior{0:02d}.pdf'.format(obs_num+1), bbox_inches='tight') + learner.model.print_hyperparameters() -if save_plots: +if save_plots and plot_type is 'pdf': pdf_pages.close() else: - fig_post, ax_p = learner.model.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, abs_y_samples, **plot_kwargs) + fig_post, ax_p, h_plotobj = learner.model.create_posterior_plot(x_test, f_true, mu_true, rel_sigma, fuv_rel, abs_y_samples, **plot_kwargs) plt.show(block=False) print "Finished!" diff --git a/read_wine_data.py b/read_wine_data.py index ba6283b..136917c 100644 --- a/read_wine_data.py +++ b/read_wine_data.py @@ -35,7 +35,7 @@ def __init__(self, data_file): log_hyp = np.log(config['hyperparameters']) n_rel = 1 -n_abs = 100 +n_abs = 300 x_rel = red_data.data.values[0:2*n_rel, 0:-1] uvi_rel = np.array([[0, 1]]) fuv_rel = red_data.data.values[0:2*n_rel, -1] @@ -51,4 +51,6 @@ def __init__(self, data_file): prefGP.set_hyperparameters(log_hyp) f = prefGP.calc_laplace(log_hyp) -log_hyp_opt = op.fmin(prefGP.calc_nlml,log_hyp) \ No newline at end of file +print np.exp(log_hyp) +log_hyp_opt = op.fmin(prefGP.calc_nlml,log_hyp) +print np.exp(log_hyp_opt) From 7903779999549dcd33eedbcce71a0200507c51fe Mon Sep 17 00:00:00 2001 From: Nicholas Lawrance Date: Mon, 23 Dec 2019 11:42:04 +0100 Subject: [PATCH 34/38] Add wine data tools --- pref_active_learning.py | 4 ++- read_wine_data.py | 66 +++++++++++++++++++++++++++-------------- 2 files changed, 47 insertions(+), 23 deletions(-) diff --git a/pref_active_learning.py b/pref_active_learning.py index 17bad23..441373e 100644 --- a/pref_active_learning.py +++ b/pref_active_learning.py @@ -1,6 +1,8 @@ # Simple 1D GP classification example import time import numpy as np +import matplotlib +matplotlib.use('Agg') import matplotlib.pyplot as plt import GPpref import plot_tools as ptt @@ -15,7 +17,7 @@ save_plots = True plot_type = 'video_frames' # 'pdf' -inter_frames = 10 +inter_frames = 30 with open('./data/shortrun_2D.yaml', 'rt') as fh: wave = yaml.safe_load(fh) diff --git a/read_wine_data.py b/read_wine_data.py index 136917c..5ac68d5 100644 --- a/read_wine_data.py +++ b/read_wine_data.py @@ -5,6 +5,8 @@ import GPpref import scipy.optimize as op +optimise_hyper = True +use_normalised = False # Get wine data: # wget -P data/wine_quality https://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-red.csv # wget -P data/wine_quality https://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-white.csv @@ -27,30 +29,50 @@ def __init__(self, data_file): ah[0].set_title('Red wine') ah[1].set_title('White wine') -plt.show(block=False) - with open('./data/statruns_wine.yaml', 'rt') as fh: config = yaml.safe_load(fh) d_x = config['GP_params']['hyper_counts'][0]-1 log_hyp = np.log(config['hyperparameters']) -n_rel = 1 -n_abs = 300 -x_rel = red_data.data.values[0:2*n_rel, 0:-1] -uvi_rel = np.array([[0, 1]]) -fuv_rel = red_data.data.values[0:2*n_rel, -1] -y_rel = np.array([1]) - -x_abs = red_data.data.values[2*n_rel:(2*n_rel+n_abs), 0:-1] -y_abs = np.expand_dims(red_data.data.values[2*n_rel:(2*n_rel+n_abs), -1]-2, axis=1).astype(dtype='int') -model_kwargs = {'x_rel': x_rel, 'uvi_rel': uvi_rel, 'x_abs': x_abs, 'y_rel': y_rel, 'y_abs': y_abs, - 'rel_kwargs': config['rel_obs_params'], 'abs_kwargs': config['abs_obs_params']} -model_kwargs.update(config['GP_params']) -model_kwargs['verbose'] = 2 -prefGP = GPpref.PreferenceGaussianProcess(**model_kwargs) - -prefGP.set_hyperparameters(log_hyp) -f = prefGP.calc_laplace(log_hyp) -print np.exp(log_hyp) -log_hyp_opt = op.fmin(prefGP.calc_nlml,log_hyp) -print np.exp(log_hyp_opt) +red_norm = (red_data.data.values - red_data.data.values.min(axis=0)) / (red_data.data.values.max(axis=0) - red_data.data.values.min(axis=0)) + +if use_normalised: + wine_data = red_norm +else: + wine_data = red_data.data.values +fh2, ah2 = plt.subplots(3, 4) +for i, aa in enumerate(ah2.flat): + aa.cla() + aa.scatter(wine_data[:,i], wine_data[:,-1], 2, wine_data[:,0]) + aa.set_title('{0}, {1}'.format(red_data.data.keys()[i], config['hyperparameters'][i])) + +fh3, ah3 = plt.subplots() +C = np.corrcoef(red_norm.T) +hmat = ah3.matshow(np.abs(C)) + +plt.show() + +if optimise_hyper: + n_rel = 1 + n_abs = 300 + x_rel = wine_data[0:2*n_rel, 0:-1] + uvi_rel = np.array([[0, 1]]) + fuv_rel = wine_data[0:2*n_rel, -1] + y_rel = np.array([1]) + + x_abs = wine_data[2*n_rel:(2*n_rel+n_abs), 0:-1] + y_abs = np.expand_dims(red_data.data.values[2*n_rel:(2*n_rel+n_abs), -1]-2, axis=1).astype(dtype='int') + model_kwargs = {'x_rel': x_rel, 'uvi_rel': uvi_rel, 'x_abs': x_abs, 'y_rel': y_rel, 'y_abs': y_abs, + 'rel_kwargs': config['rel_obs_params'], 'abs_kwargs': config['abs_obs_params']} + model_kwargs.update(config['GP_params']) + model_kwargs['verbose'] = 2 + prefGP = GPpref.PreferenceGaussianProcess(**model_kwargs) + + prefGP.set_hyperparameters(log_hyp) + f = prefGP.calc_laplace(log_hyp) + print np.exp(log_hyp) + log_hyp_opt = op.fmin(prefGP.calc_nlml,log_hyp) + print np.exp(log_hyp_opt) + config['hyperparameters'] = np.exp(log_hyp_opt).tolist() + with open('data/wine_quality/learned_red_params.yaml', 'w') as fh: + yaml.safe_dump(config, fh) From 85e08ac951d46ae6ce0e1cce97c4a003514b9816 Mon Sep 17 00:00:00 2001 From: Nicholas Lawrance Date: Mon, 23 Dec 2019 15:35:56 +0100 Subject: [PATCH 35/38] Add option for random test point sampling --- active_statruns.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/active_statruns.py b/active_statruns.py index c58644f..eadc825 100644 --- a/active_statruns.py +++ b/active_statruns.py @@ -56,7 +56,17 @@ # True function x_plot = np.linspace(0.0, 1.0, statrun_params['n_xtest'], dtype='float') -x_test = ptt.make_meshlist(x_plot, d_x) + +try: + if statrun_params['sample_type'] is 'uniform': + x_test = ptt.make_meshlist(x_plot, d_x) + else: + x_test = np.random.uniform(0.0, 1.0, + size = (statrun_params['n_xtest'], d_x)) +except KeyError: + print('Config does not contain statrun_params: sample_type value. Default to uniform sampling') + x_test = ptt.make_meshlist(x_plot, d_x) + far_domain = np.tile(np.array([[-3.0], [-2.0]]), d_x) # Construct active learner objects From 24777e2cd52ceb264b8422ed7dcee6b840b8f601 Mon Sep 17 00:00:00 2001 From: Nicholas Lawrance Date: Mon, 23 Dec 2019 16:52:49 +0100 Subject: [PATCH 36/38] Fix OrdinalSampler abs sample --- active_learners.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/active_learners.py b/active_learners.py index cec0103..7cd49f4 100644 --- a/active_learners.py +++ b/active_learners.py @@ -707,7 +707,7 @@ def select_observation(self, domain=None, x_test=None, n_test=50, y_threshold=5, V_abs += p_obs*self.test_observation(x, y_obs+1, None, x_test, y_threshold) if V_abs > V_max: V_max = V_abs - x_best = x + x_best = np.array([x]) if self.verbose >= 1: print 'V_max_abs = {0}, t = {1}s'.format(V_max, time.time() - t_rel) From 9f4ff52ac56787ede8c64ac32e0d9a6303629a0d Mon Sep 17 00:00:00 2001 From: Nicholas Lawrance Date: Fri, 10 Jan 2020 10:04:52 +0100 Subject: [PATCH 37/38] Wine data downloader --- read_wine_data.py | 57 +++++++++++++++++++++++----------------- utils/__init__.py | 0 utils/data_downloader.py | 42 +++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 24 deletions(-) create mode 100644 utils/__init__.py create mode 100644 utils/data_downloader.py diff --git a/read_wine_data.py b/read_wine_data.py index 5ac68d5..a6588d7 100644 --- a/read_wine_data.py +++ b/read_wine_data.py @@ -4,53 +4,62 @@ import yaml import GPpref import scipy.optimize as op +from utils.data_downloader import MrDataGrabber -optimise_hyper = True -use_normalised = False -# Get wine data: -# wget -P data/wine_quality https://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-red.csv -# wget -P data/wine_quality https://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-white.csv -class WineQualityData(object): +class WineQualityData(object): def __init__(self, data_file): self.file = data_file self.data = pandas.read_csv(self.file, delimiter=';') -red_data = WineQualityData('data/wine_quality/winequality-red.csv') -white_data = WineQualityData('data/wine_quality/winequality-white.csv') +optimise_hyper = True +use_normalised = True +R_limit = 0.1 +wine_type = 'white' -fh, ah = plt.subplots(1, 2) -ah[0].hist(red_data.data.quality, np.arange(-0.5, 11, 1.0)) -ah[1].hist(white_data.data.quality, np.arange(-0.5, 11, 1.0)) -for a in ah: - a.set_xticks(np.arange(11)) - a.set_xlabel('Score') -ah[0].set_ylabel('Count') -ah[0].set_title('Red wine') -ah[1].set_title('White wine') +# Get wine data: +url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-{0}.csv'.format(wine_type) +file_loc = 'data/wine_quality/' +download_data = MrDataGrabber(url, file_loc) +download_data.download() + +input_data = WineQualityData(download_data.target_file) + +fh, ah = plt.subplots(1, 1) +ah.hist(input_data.data.quality, np.arange(-0.5, 11, 1.0)) +ah.set_xticks(np.arange(11)) +ah.set_xlabel('Score') +ah.set_ylabel('Count') +ah.set_title('{0} wine'.format(wine_type)) with open('./data/statruns_wine.yaml', 'rt') as fh: config = yaml.safe_load(fh) d_x = config['GP_params']['hyper_counts'][0]-1 log_hyp = np.log(config['hyperparameters']) -red_norm = (red_data.data.values - red_data.data.values.min(axis=0)) / (red_data.data.values.max(axis=0) - red_data.data.values.min(axis=0)) +wine_norm = (input_data.data.values - input_data.data.values.min(axis=0)) / (input_data.data.values.max(axis=0) - input_data.data.values.min(axis=0)) if use_normalised: - wine_data = red_norm + wine_data = wine_norm else: - wine_data = red_data.data.values + wine_data = input_data.data.values fh2, ah2 = plt.subplots(3, 4) for i, aa in enumerate(ah2.flat): aa.cla() aa.scatter(wine_data[:,i], wine_data[:,-1], 2, wine_data[:,0]) - aa.set_title('{0}, {1}'.format(red_data.data.keys()[i], config['hyperparameters'][i])) + aa.set_title('{0}, {1}'.format(input_data.data.keys()[i], config['hyperparameters'][i])) fh3, ah3 = plt.subplots() -C = np.corrcoef(red_norm.T) +C = np.corrcoef(wine_norm, rowvar=False) hmat = ah3.matshow(np.abs(C)) plt.show() +for R, var_name in zip(C[0:-1,-1], input_data.data.keys()): + print('{0}: R = {1:0.2f}'.format(var_name, R)) +corr_variables = (abs(C[-1]) >= R_limit) + +# Subsample wine data down to only correlated variables +# wine_data = wine_data[:, corr_variables] if optimise_hyper: n_rel = 1 @@ -61,7 +70,7 @@ def __init__(self, data_file): y_rel = np.array([1]) x_abs = wine_data[2*n_rel:(2*n_rel+n_abs), 0:-1] - y_abs = np.expand_dims(red_data.data.values[2*n_rel:(2*n_rel+n_abs), -1]-2, axis=1).astype(dtype='int') + y_abs = np.expand_dims(input_data.data.values[2 * n_rel:(2 * n_rel + n_abs), -1] - 2, axis=1).astype(dtype='int') model_kwargs = {'x_rel': x_rel, 'uvi_rel': uvi_rel, 'x_abs': x_abs, 'y_rel': y_rel, 'y_abs': y_abs, 'rel_kwargs': config['rel_obs_params'], 'abs_kwargs': config['abs_obs_params']} model_kwargs.update(config['GP_params']) @@ -74,5 +83,5 @@ def __init__(self, data_file): log_hyp_opt = op.fmin(prefGP.calc_nlml,log_hyp) print np.exp(log_hyp_opt) config['hyperparameters'] = np.exp(log_hyp_opt).tolist() - with open('data/wine_quality/learned_red_params.yaml', 'w') as fh: + with open('data/wine_quality/learned_{0}_params.yaml'.format(wine_type), 'w') as fh: yaml.safe_dump(config, fh) diff --git a/utils/__init__.py b/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/utils/data_downloader.py b/utils/data_downloader.py new file mode 100644 index 0000000..0afadfa --- /dev/null +++ b/utils/data_downloader.py @@ -0,0 +1,42 @@ +from __future__ import print_function +import os +import urllib3 +import zipfile + +class MrDataGrabber(object): # Does not work with bananas + + def __init__(self, url, target_path): + self.url = url + self.path = target_path + self.target_basename = os.path.basename(self.url) + self.target_file = os.path.join(self.path, self.target_basename) + + def download(self): + if os.path.exists(self.target_file): + print('Target file already downloaded, please remove if you wish to update.') + else: + http = urllib3.PoolManager() + r = http.request('GET', self.url, preload_content=False) + + with open(self.target_file, 'wb') as out: + print('Downloading {0}...'.format(self.url), end='') + while True: + data = r.read(2**16) + if not data: + break + out.write(data) + r.release_conn() + print(' done.') + + if os.path.splitext(self.target_basename)[1].lower() == '.zip': + self.unzip_data() + return self.target_file + + def unzip_data(self, target_dir=None, force=False): + if target_dir is None: + target_dir = os.path.join(self.path, os.path.splitext(self.target_basename)[0]) + if (not force) and os.path.isdir(target_dir): + print('Target directory already exists, not unzipping.') + else: + zip_ref = zipfile.ZipFile(self.target_file, 'r') + zip_ref.extractall(target_dir) From bc11f5aa5eb7f2cb5c849736fb335eaa504d8a87 Mon Sep 17 00:00:00 2001 From: Nicholas Lawrance Date: Fri, 10 Jan 2020 11:04:33 +0100 Subject: [PATCH 38/38] Move wine data reader to utils --- read_wine_data.py | 17 +++-------------- utils/wine_data.py | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 14 deletions(-) create mode 100644 utils/wine_data.py diff --git a/read_wine_data.py b/read_wine_data.py index a6588d7..f79d409 100644 --- a/read_wine_data.py +++ b/read_wine_data.py @@ -1,29 +1,18 @@ import numpy as np -import pandas import matplotlib.pyplot as plt import yaml import GPpref import scipy.optimize as op -from utils.data_downloader import MrDataGrabber +from utils.wine_data import WineQualityData -class WineQualityData(object): - def __init__(self, data_file): - self.file = data_file - self.data = pandas.read_csv(self.file, delimiter=';') - -optimise_hyper = True +optimise_hyper = False use_normalised = True R_limit = 0.1 wine_type = 'white' # Get wine data: -url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-{0}.csv'.format(wine_type) -file_loc = 'data/wine_quality/' -download_data = MrDataGrabber(url, file_loc) -download_data.download() - -input_data = WineQualityData(download_data.target_file) +input_data = WineQualityData(wine_type=wine_type) fh, ah = plt.subplots(1, 1) ah.hist(input_data.data.quality, np.arange(-0.5, 11, 1.0)) diff --git a/utils/wine_data.py b/utils/wine_data.py new file mode 100644 index 0000000..fae5562 --- /dev/null +++ b/utils/wine_data.py @@ -0,0 +1,21 @@ +import pandas +import os +from utils.data_downloader import MrDataGrabber + + +class WineQualityData(object): + def __init__(self, wine_type='red', file_loc='data/wine_quality/'): + url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-{0}.csv'.format( + wine_type) + + try: + # Create target Directory + os.mkdir(file_loc) + print("Directory ", file_loc, " Created ") + except OSError: + print("Directory ", file_loc, " already exists") + + self.download_data = MrDataGrabber(url, file_loc) + self.download_data.download() + self.file = self.download_data.target_file + self.data = pandas.read_csv(self.file, delimiter=';')