From 1f5b74eb7cb9ade606b0657d1ad8b7ec11beab9e Mon Sep 17 00:00:00 2001 From: William de Vazelhes Date: Mon, 2 Jul 2018 16:24:20 +0200 Subject: [PATCH 01/11] FIX fixes #88 Stores L and G in addition to what was stored before --- metric_learn/lmnn.py | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/metric_learn/lmnn.py b/metric_learn/lmnn.py index dea12f0c..04a69396 100644 --- a/metric_learn/lmnn.py +++ b/metric_learn/lmnn.py @@ -95,12 +95,16 @@ def fit(self, X, y): L = self.L_ objective = np.inf + # we initialize the roll back + L_old = L.copy() + G_old = G.copy() + df_old = df.copy() + a1_old = [a.copy() for a in a1] + a2_old = [a.copy() for a in a2] + objective_old = objective + # main loop for it in xrange(1, self.max_iter): - df_old = df.copy() - a1_old = [a.copy() for a in a1] - a2_old = [a.copy() for a in a2] - objective_old = objective # Compute pairwise distances under current metric Lx = L.dot(self.X_.T).T g0 = _inplace_paired_L2(*Lx[impostors]) @@ -158,14 +162,25 @@ def fit(self, X, y): if delta_obj > 0: # we're getting worse... roll back! learn_rate /= 2.0 + L = L_old + G = G_old df = df_old a1 = a1_old a2 = a2_old objective = objective_old else: - # update L - L -= learn_rate * 2 * L.dot(G) - learn_rate *= 1.01 + # We did good. We store this point as reference in case we do + # worse next time. + objective_old = objective + L_old = L.copy() + G_old = G.copy() + df_old = df.copy() + a1_old = [a.copy() for a in a1] + a2_old = [a.copy() for a in a2] + # we update L and will see in the next iteration if it does indeed + # better + L -= learn_rate * 2 * L.dot(G) + learn_rate *= 1.01 # check for convergence if it > self.min_iter and abs(delta_obj) < self.convergence_tol: @@ -177,7 +192,7 @@ def fit(self, X, y): print("LMNN didn't converge in %d steps." % self.max_iter) # store the last L - self.L_ = L + self.L_ = L_old self.n_iter_ = it return self From a18fa137749d5af5ad5aef8e1f19fe219df93cbc Mon Sep 17 00:00:00 2001 From: William de Vazelhes Date: Tue, 3 Jul 2018 12:17:01 +0200 Subject: [PATCH 02/11] TST: non regression test for this PR - test that LMNN converges on a simple example where it should converge - test that the objective function never has twice the same value --- test/metric_learn_test.py | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/test/metric_learn_test.py b/test/metric_learn_test.py index 6d78c657..6e890022 100644 --- a/test/metric_learn_test.py +++ b/test/metric_learn_test.py @@ -1,8 +1,11 @@ import unittest +import re +import sys import numpy as np from six.moves import xrange +from sklearn.externals.six import StringIO from sklearn.metrics import pairwise_distances -from sklearn.datasets import load_iris +from sklearn.datasets import load_iris, make_classification from numpy.testing import assert_array_almost_equal from metric_learn import ( @@ -70,6 +73,40 @@ def test_iris(self): csep = class_separation(lmnn.transform(), self.iris_labels) self.assertLess(csep, 0.25) + def test_convergence_simple_example(self): + # LMNN should converge on this simple example, which it did not with + # this issue: https://github.com/metric-learn/metric-learn/issues/88 + X, y = make_classification(random_state=0) + old_stdout = sys.stdout + sys.stdout = StringIO() + lmnn = LMNN(verbose=True) + try: + lmnn.fit(X, y) + finally: + out = sys.stdout.getvalue() + sys.stdout.close() + sys.stdout = old_stdout + assert ("LMNN converged with objective" in out) + + def test_no_twice_same_objective(self): + # test that the objective function never has twice the same value + # see https://github.com/metric-learn/metric-learn/issues/88 + X, y = make_classification(random_state=0) + old_stdout = sys.stdout + sys.stdout = StringIO() + lmnn = LMNN(verbose=True) + try: + lmnn.fit(X, y) + finally: + out = sys.stdout.getvalue() + sys.stdout.close() + sys.stdout = old_stdout + lines = re.split("\n+", out) + objectives = [re.search("\d* (?:(\d*.\d*))[ | -]\d*.\d*", s) + for s in lines] + objectives = [match.group(1) for match in objectives if match is not None] + assert len(objectives) == len(set(objectives)) + class TestSDML(MetricTestCase): def test_iris(self): From 5fb5ec2eb6e111ee4853ac46c06f7e5f7b76a816 Mon Sep 17 00:00:00 2001 From: William de Vazelhes Date: Fri, 20 Jul 2018 18:17:23 +0200 Subject: [PATCH 03/11] MAINT: Invert the order of algorithm: Try forward updates rather than doing rollback after wrong updates --- keyring.deb | Bin 0 -> 3258 bytes metric_learn/lmnn.py | 172 +++++++++++++++++++------------------- test/metric_learn_test.py | 3 +- 3 files changed, 86 insertions(+), 89 deletions(-) create mode 100644 keyring.deb diff --git a/keyring.deb b/keyring.deb new file mode 100644 index 0000000000000000000000000000000000000000..cc48860b1cf8bc94f6c05644e7a630f337ada86e GIT binary patch literal 3258 zcmai1c{J1w8$~51`&c5fCoRligko%2l6~LJgwc$lXl58OC?c;VLWn6!Y3yoL)*3IN z31i=tt;N2KZTNkC-}&D6eE)psz2`aioadf%pTF+?Ssjz3;6;rYQeW%+#cBIGIC*;e!?!6=S+zpWvJ+uBG{;!uKJBwtpI%oWA|~Tc zK@Bubxb;~7mcMu1y`SU~R`iyqYX|3s!5}<&da{CKh#w@UDOiYU-TFnkal|LAnG<~6 z55;lUn>SOHL-&<&IA-Yduw$?K>l<#Y3f77K3r8n_tyOZ`j0zx_C<(N6D^6paK7f`5 zQX8Wg)yqzg;uW@xzM0DEIM-)!z7@~JcI9$L`c|tm5254Wl{oJyLf|>}rrhLw`efA6 zIT&$r_s(24eNeZg3Zytar z?+N8eR7M+Ce*vH%|BO4{>|eVg&^PHCqSfgc@Drte{V~ntC+x-ST`}FYoX=Wmc(gzP zS5=c;SJ6anj;{EdMR#V$0A#ud)lNDqq)Xl9-bs}sW|OXf+W@G;{d1gd{uZ_&bzree zsRjZaTD-DX!V?xr9!K8$gE)HD$6I8kV86};5cA~5WDM=ktcAVfQRW6)J@IHAR#WXb zt=s{zkcRa0Q*HV=SI79IY^`LUhG&UtTOV%^$jNrKE!HPd*Bff_+?_=`(nRdSoIhVD z(75hxJng+kO4mL}ecCH#AbZgQ%+NF;Mq91!1ng#sx6iY+->rm}5Bb=7cVdNWpTcBX zYR%b&Xz$lXlzE5Z(G~IjY;leekrCxBq>z@Cz=8)?MeIL5u;ey1BiIWqgm2Hx4kzMP zuu71+HuDDhX~`WDy$MZT+wan4WoX=DT|AHiOsBj*AZWm~?bMgkldH?ya6aH3FOWJl zSs}HUffvkPvbG7MVi1L+=Z?l6anc zi)nt9aP%}!t%^*W%)QepVBF(ZI1B-e63!pldQO?33|82C?&E-QV2|5B!J5xDMLTns znz@eHqw;5%`+|Br4&>Y0=(AFXjW1tiN60~qxD2=uH!8QA=zC@RBs(tuKxps6v2}j1 zvWCzK9Csiti>y&?hhHu1js{9LRYf~gkK|pRx_g7`Tus+6a-bFG0h~gzeUx^Fbt+9t zWrkZTxb@%(Ng{5I%*rs+Sm$Tu(dO2+tq&i-wU63J+%L#fEtZ8Qd@GXN(;fP92gBN= zkfJ3$(YRarVzrWg^t}G&)tly9vGD%3*G}_byrtt@`WhBOT5J0}nD6%X5PuNpHaPy$ zL-h!quozLJ8YWCR#SxXt(%5*hk0)W z?vD&KEK>U2)rzx4PVCa8`Z4@N3*f^VIk{xsla#jw&$@I*aJg_J++HFoqR{P2$zw8N zKiyPWG^OI^ZDB7cMsY8tE)|8ThAC^K{VM$5otRTRABFKmPMc%cX(^4qGq(Hpja-`C zay|@P@pwq=`WTnR$DQ@ycm>I9#V(kJYzr?bZyl~Wq*l-fR3l^fG#i`j6lZO12P~(3 zUzG80%D1|q1o9uZ<2x?};$6^&{1xi)EyZsHFoa^U*i;xCrX^JcDz)_DM!PTyM=yK$?cFq6+EhnLYxVF(gyt`|47e7Y ze7(B)Jqhnw!RR*C^H)?^z|$vS*hJ0X{I5HPpSCCBDWRs_@?SjT%Ib)1_M#!)kJL)M zHjVf)sfuTw$7P%~?bJLk;U|Z+)7g-hj=L~3kL0zoyb?jmYSU zVD^l;-&jG~T+A&McV6WoRp!P<7v|Gt!tw}y&R=jzyW>H25B5g~_$|>be;Db(&~2;ol-)c_7AU#a zY+dF$DV-d)`i!}AgK)0t>aVCVj(dgALJh9}$nL0|)ckp-sao%&_a%d-I5ee{Vzldl z@J^QeZq0uNu02!cA7au2ul_Wldx}T=B@9a_d+w|AC8EK1ENU7fs`d_m7zArTk4j@h zbTD)L@*PDpG7IR>3z$-^q}8VKu;CHnFy=W=2AS~D)OpY<6Gt6R8yuuxpUH3x`GHXy z<)o2k&7HYUkz12L#kVm(7m2-FELR~u6&rPrsdqN)zBp^oa8fUg@8_Xxyje%HvkQ9AdH z%tW?TaG#ae-h#)I4^Iew-pv6;ULhhJ5tj(;j5zIToun9{xRf2qn$q?N$B7>2tduvZ z}Hi~J{U804wX zCG*Y>;Eutj4 zSY3DCaz)j_4j9w4BZk1l?g)EZTk@M)P94=Q)zS7I{M8sql*^y*gzFw_NSUgtWSS&U z!}_{uCfgl%UfOfh&q{dp!nUpG6UmqR#{^~*?eUFnD$|jufsJ1sy93yh8{ub&t2k0w zjF0Qhk*Ukc-|4+O$~<)QBAi%~;6M^Bv$&xgnqWY|KZ{?gt46;qT=kvN$=LKmSER&CTkFrTcoHQ>f+{+M;^0)SJw& z$7yPleJA9vMbWc~1sbm>xsxA6k@M3oyig?x_GO(3{NeE)DpjMUL`W|elW4~~nN9ie z>PF2rkK!Y0nS>Lt^Bi_uJoVx38yrp`cc-JO^^~xPUW|FhCfcgOx-dPMZg$_{p`JoY zrV1fZO{3$zo1MdX2_1#Y;vKCb;Jm(oM%VtX>SJ9lFVt1u7z@-$dEBbv)3ul;BL1Xh z_K2IGyfW6b*rKRJG;GX9jwWBI#QJ|>w&yG}wa~QL;A8xROcBHWjSWrVFK=$s;(5)( zOiCj(zaW*omSbC#f1Cx2jbC%YiQPLQUKYx85#KQK2u~#re0^<0VI{2`deb;fFG>xT z${!*tAqpSdn3rqfT6DD-y*nbmR2<&<-oay)X|ZeErOPO*$Gl#lt-nhwIlq454^W4r zK4Sv_19yyOr1-W{FE>sm0LDVevC-xlYJ#VFPeW!cSxR=wG6&X23Oow9(t`o&&-tlu zehRUYhY>-1-Kw>BxIlc-$D2wyJKBs7Ym8*>2j@aS4v9-L^FooFoi8d~(@5K+x@+pOl4Fm# z0#^E>a_51Dy43u`2PwPKaiB&3P6kq4Sw^gaU1_>xya9_0(% 1: - plus1 = act1 & ~a1[nn_idx] - minus1 = a1[nn_idx] & ~act1 - plus2 = act2 & ~a2[nn_idx] - minus2 = a2[nn_idx] & ~act2 - else: - plus1 = act1 - plus2 = act2 - minus1 = np.zeros(0, dtype=int) - minus2 = np.zeros(0, dtype=int) - - targets = target_neighbors[:,nn_idx] - PLUS, pweight = _count_edges(plus1, plus2, impostors, targets) - df += _sum_outer_products(self.X_, PLUS[:,0], PLUS[:,1], pweight) - MINUS, mweight = _count_edges(minus1, minus2, impostors, targets) - df -= _sum_outer_products(self.X_, MINUS[:,0], MINUS[:,1], mweight) - - in_imp, out_imp = impostors - df += _sum_outer_products(self.X_, in_imp[minus1], out_imp[minus1]) - df += _sum_outer_products(self.X_, in_imp[minus2], out_imp[minus2]) - - df -= _sum_outer_products(self.X_, in_imp[plus1], out_imp[plus1]) - df -= _sum_outer_products(self.X_, in_imp[plus2], out_imp[plus2]) - - a1[nn_idx] = act1 - a2[nn_idx] = act2 - - # do the gradient update - assert not np.isnan(df).any() - G = dfG * reg + df * (1-reg) - - # compute the objective function - objective = total_active * (1-reg) - objective += G.flatten().dot(L.T.dot(L).flatten()) - assert not np.isnan(objective) - delta_obj = objective - objective_old + + # first iteration + G, objective, total_active, df, a1, a2 = ( + self._loss_grad(L, dfG, impostors, 1, k, reg, target_neighbors, df, a1, + a2)) + for it in xrange(2, self.max_iter): + # we first try to find a value of L that has better objective than the + # previous L, following the gradient: + + # we want to enter the loop for the first try, with the original + # learning rate (hence * 2 since it will be / 2) + delta_obj = 1 + learn_rate *= 2 + while delta_obj > 0: + # we keep looking until the L has a better objective, retrying with a + # smaller update if necessary (reducing the learning rate) + learn_rate /= 2 + # the next point next_L is found by a gradient step + L_next = L - 2 * learn_rate * G + # we compute the objective at next point + # we copy variables that can be modified by _loss_grad, because if we + # retry we don t want to modify them several times + (G_next, objective_next, total_active_next, df_next, a1_next, + a2_next) = ( + self._loss_grad(L_next, dfG, impostors, it, k, reg, + target_neighbors, df.copy(), a1.copy(), a2.copy())) + assert not np.isnan(objective) + delta_obj = objective_next - objective + # when the good L is found, we start from here before doing next + # iteration, and we increase slightly the learning rate + L = L_next + G, df, objective, total_active, a1, a2 = ( + G_next, df_next, objective_next, total_active_next, a1_next, a2_next) + learn_rate *= 1.01 if self.verbose: print(it, objective, delta_obj, total_active, learn_rate) - # update step size - if delta_obj > 0: - # we're getting worse... roll back! - learn_rate /= 2.0 - L = L_old - G = G_old - df = df_old - a1 = a1_old - a2 = a2_old - objective = objective_old - else: - # We did good. We store this point as reference in case we do - # worse next time. - objective_old = objective - L_old = L.copy() - G_old = G.copy() - df_old = df.copy() - a1_old = [a.copy() for a in a1] - a2_old = [a.copy() for a in a2] - # we update L and will see in the next iteration if it does indeed - # better - L -= learn_rate * 2 * L.dot(G) - learn_rate *= 1.01 - # check for convergence if it > self.min_iter and abs(delta_obj) < self.convergence_tol: if self.verbose: @@ -192,10 +140,58 @@ def fit(self, X, y): print("LMNN didn't converge in %d steps." % self.max_iter) # store the last L - self.L_ = L_old + self.L_ = L self.n_iter_ = it return self + def _loss_grad(self, L, dfG, impostors, it, k, reg, target_neighbors, df, a1, + a2): + # Compute pairwise distances under current metric + Lx = L.dot(self.X_.T).T + g0 = _inplace_paired_L2(*Lx[impostors]) + Ni = 1 + _inplace_paired_L2(Lx[target_neighbors], Lx[:, None, :]) + g1, g2 = Ni[impostors] + # compute the gradient + total_active = 0 + for nn_idx in reversed(xrange(k)): + act1 = g0 < g1[:, nn_idx] + act2 = g0 < g2[:, nn_idx] + total_active += act1.sum() + act2.sum() + + if it > 1: + plus1 = act1 & ~a1[nn_idx] + minus1 = a1[nn_idx] & ~act1 + plus2 = act2 & ~a2[nn_idx] + minus2 = a2[nn_idx] & ~act2 + else: + plus1 = act1 + plus2 = act2 + minus1 = np.zeros(0, dtype=int) + minus2 = np.zeros(0, dtype=int) + + targets = target_neighbors[:, nn_idx] + PLUS, pweight = _count_edges(plus1, plus2, impostors, targets) + df += _sum_outer_products(self.X_, PLUS[:, 0], PLUS[:, 1], pweight) + MINUS, mweight = _count_edges(minus1, minus2, impostors, targets) + df -= _sum_outer_products(self.X_, MINUS[:, 0], MINUS[:, 1], mweight) + + in_imp, out_imp = impostors + df += _sum_outer_products(self.X_, in_imp[minus1], out_imp[minus1]) + df += _sum_outer_products(self.X_, in_imp[minus2], out_imp[minus2]) + + df -= _sum_outer_products(self.X_, in_imp[plus1], out_imp[plus1]) + df -= _sum_outer_products(self.X_, in_imp[plus2], out_imp[plus2]) + + a1[nn_idx] = act1 + a2[nn_idx] = act2 + # do the gradient update + assert not np.isnan(df).any() + G = dfG * reg + df * (1 - reg) + # compute the objective function + objective = total_active * (1 - reg) + objective += G.flatten().dot(L.T.dot(L).flatten()) + return G, objective, total_active, df, a1, a2 + def _select_targets(self): target_neighbors = np.empty((self.X_.shape[0], self.k), dtype=int) for label in self.labels_: diff --git a/test/metric_learn_test.py b/test/metric_learn_test.py index 6e890022..f1db3fe2 100644 --- a/test/metric_learn_test.py +++ b/test/metric_learn_test.py @@ -105,7 +105,8 @@ def test_no_twice_same_objective(self): objectives = [re.search("\d* (?:(\d*.\d*))[ | -]\d*.\d*", s) for s in lines] objectives = [match.group(1) for match in objectives if match is not None] - assert len(objectives) == len(set(objectives)) + # we remove the last element because it can be equal to the penultimate + assert len(objectives[:-1]) == len(set(objectives[:-1])) class TestSDML(MetricTestCase): From 8f5f62d52d77974e61dc3fb934a72e79784f9f87 Mon Sep 17 00:00:00 2001 From: William de Vazelhes Date: Fri, 20 Jul 2018 19:52:36 +0200 Subject: [PATCH 04/11] MAINT: update code according to comments https://github.com/metric-learn/metric-learn/pull/101#pullrequestreview-134072011 --- test/metric_learn_test.py | 45 ++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/test/metric_learn_test.py b/test/metric_learn_test.py index f1db3fe2..b8b45980 100644 --- a/test/metric_learn_test.py +++ b/test/metric_learn_test.py @@ -79,34 +79,35 @@ def test_convergence_simple_example(self): X, y = make_classification(random_state=0) old_stdout = sys.stdout sys.stdout = StringIO() - lmnn = LMNN(verbose=True) + lmnn = python_LMNN(verbose=True) try: lmnn.fit(X, y) finally: out = sys.stdout.getvalue() sys.stdout.close() sys.stdout = old_stdout - assert ("LMNN converged with objective" in out) - - def test_no_twice_same_objective(self): - # test that the objective function never has twice the same value - # see https://github.com/metric-learn/metric-learn/issues/88 - X, y = make_classification(random_state=0) - old_stdout = sys.stdout - sys.stdout = StringIO() - lmnn = LMNN(verbose=True) - try: - lmnn.fit(X, y) - finally: - out = sys.stdout.getvalue() - sys.stdout.close() - sys.stdout = old_stdout - lines = re.split("\n+", out) - objectives = [re.search("\d* (?:(\d*.\d*))[ | -]\d*.\d*", s) - for s in lines] - objectives = [match.group(1) for match in objectives if match is not None] - # we remove the last element because it can be equal to the penultimate - assert len(objectives[:-1]) == len(set(objectives[:-1])) + assert "LMNN converged with objective" in out + + +def test_no_twice_same_objective(capsys): + # test that the objective function never has twice the same value + # see https://github.com/metric-learn/metric-learn/issues/88 + X, y = make_classification(random_state=0) + lmnn = python_LMNN(verbose=True) + lmnn.fit(X, y) + out, err = capsys.readouterr() + lines = re.split("\n+", out) + # we get only objectives from each line: + # the regexp matches a float that follows an integer (the iteration + # number), and which is followed by a (signed) float (delta obj). It + # matches for instance:   + # 3 **1113.7665747189938** -3.182774197440267 46431.0200999999999998e-06 + objectives = [re.search("\d* (?:(\d*.\d*))[ | -]\d*.\d*", s) + for s in lines] + objectives = [match.group(1) for match in objectives if match is not None] + # we remove the last element because it can be equal to the penultimate + # if the last gradient update is null + assert len(objectives[:-1]) == len(set(objectives[:-1])) class TestSDML(MetricTestCase): From ec7e4976b17e901af4cb711932ccaee171c8a67c Mon Sep 17 00:00:00 2001 From: William de Vazelhes Date: Fri, 20 Jul 2018 20:06:57 +0200 Subject: [PATCH 05/11] FIX: update also test_convergence_simple_example --- test/metric_learn_test.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/test/metric_learn_test.py b/test/metric_learn_test.py index ac4dd350..4b713b9f 100644 --- a/test/metric_learn_test.py +++ b/test/metric_learn_test.py @@ -73,20 +73,15 @@ def test_iris(self): csep = class_separation(lmnn.transform(), self.iris_labels) self.assertLess(csep, 0.25) - def test_convergence_simple_example(self): - # LMNN should converge on this simple example, which it did not with - # this issue: https://github.com/metric-learn/metric-learn/issues/88 - X, y = make_classification(random_state=0) - old_stdout = sys.stdout - sys.stdout = StringIO() - lmnn = python_LMNN(verbose=True) - try: - lmnn.fit(X, y) - finally: - out = sys.stdout.getvalue() - sys.stdout.close() - sys.stdout = old_stdout - assert "LMNN converged with objective" in out + +def test_convergence_simple_example(capsys): + # LMNN should converge on this simple example, which it did not with + # this issue: https://github.com/metric-learn/metric-learn/issues/88 + X, y = make_classification(random_state=0) + lmnn = python_LMNN(verbose=True) + lmnn.fit(X, y) + out, _ = capsys.readouterr() + assert "LMNN converged with objective" in out def test_no_twice_same_objective(capsys): @@ -95,7 +90,7 @@ def test_no_twice_same_objective(capsys): X, y = make_classification(random_state=0) lmnn = python_LMNN(verbose=True) lmnn.fit(X, y) - out, err = capsys.readouterr() + out, _ = capsys.readouterr() lines = re.split("\n+", out) # we get only objectives from each line: # the regexp matches a float that follows an integer (the iteration From 6cc3984d58f0deecc3cc1c9f4875d0d2af83247f Mon Sep 17 00:00:00 2001 From: William de Vazelhes Date: Tue, 31 Jul 2018 13:40:35 +0200 Subject: [PATCH 06/11] FIX: remove \xc2 character --- test/metric_learn_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/metric_learn_test.py b/test/metric_learn_test.py index 4b713b9f..eb7ebb62 100644 --- a/test/metric_learn_test.py +++ b/test/metric_learn_test.py @@ -94,8 +94,8 @@ def test_no_twice_same_objective(capsys): lines = re.split("\n+", out) # we get only objectives from each line: # the regexp matches a float that follows an integer (the iteration - # number), and which is followed by a (signed) float (delta obj). It - # matches for instance:   + # number), and which is followed by a (signed) float (delta obj). It + # matches for instance: # 3 **1113.7665747189938** -3.182774197440267 46431.0200999999999998e-06 objectives = [re.search("\d* (?:(\d*.\d*))[ | -]\d*.\d*", s) for s in lines] From 754de7ada82096d10bc23a5f7034dc8d7280de16 Mon Sep 17 00:00:00 2001 From: William de Vazelhes Date: Tue, 31 Jul 2018 15:35:53 +0200 Subject: [PATCH 07/11] FIX: use list to copy list for python2 compatibility --- metric_learn/lmnn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metric_learn/lmnn.py b/metric_learn/lmnn.py index 9de433be..67d4a116 100644 --- a/metric_learn/lmnn.py +++ b/metric_learn/lmnn.py @@ -117,7 +117,7 @@ def fit(self, X, y): (G_next, objective_next, total_active_next, df_next, a1_next, a2_next) = ( self._loss_grad(L_next, dfG, impostors, it, k, reg, - target_neighbors, df.copy(), a1.copy(), a2.copy())) + target_neighbors, df.copy(), list(a1), list(a2))) assert not np.isnan(objective) delta_obj = objective_next - objective # when the good L is found, we start from here before doing next From dbb1ec8599df802385ef0c64173e62de4e4c43ae Mon Sep 17 00:00:00 2001 From: William de Vazelhes Date: Mon, 6 Aug 2018 14:04:36 +0200 Subject: [PATCH 08/11] MAINT: make code more readable with while break (see https://github.com/metric-learn/metric-learn/pull/101#discussion_r206998425) --- metric_learn/lmnn.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/metric_learn/lmnn.py b/metric_learn/lmnn.py index 67d4a116..19222574 100644 --- a/metric_learn/lmnn.py +++ b/metric_learn/lmnn.py @@ -93,23 +93,17 @@ def fit(self, X, y): # initialize L L = self.L_ - # first iteration + # first iteration: we compute variables (including objective and gradient) + # at initialization point G, objective, total_active, df, a1, a2 = ( self._loss_grad(L, dfG, impostors, 1, k, reg, target_neighbors, df, a1, a2)) + for it in xrange(2, self.max_iter): - # we first try to find a value of L that has better objective than the - # previous L, following the gradient: - - # we want to enter the loop for the first try, with the original - # learning rate (hence * 2 since it will be / 2) - delta_obj = 1 - learn_rate *= 2 - while delta_obj > 0: - # we keep looking until the L has a better objective, retrying with a - # smaller update if necessary (reducing the learning rate) - learn_rate /= 2 - # the next point next_L is found by a gradient step + # then at each iteration, we try to find a value of L that has better + # objective than the previous L, following the gradient: + while True: + # the next point next_L to try out is found by a gradient step L_next = L - 2 * learn_rate * G # we compute the objective at next point # we copy variables that can be modified by _loss_grad, because if we @@ -120,8 +114,17 @@ def fit(self, X, y): target_neighbors, df.copy(), list(a1), list(a2))) assert not np.isnan(objective) delta_obj = objective_next - objective - # when the good L is found, we start from here before doing next - # iteration, and we increase slightly the learning rate + if delta_obj > 0: + # if we did not find a better objective, we retry with an L closer to + # the starting point, by decreasing the learning rate (making the + # gradient step smaller) + learn_rate /= 2 + else: + # otherwise, if we indeed found a better obj, we get out of the loop + break + # when the better L is found (and the related variables), we set the + # old variables to these new ones before next iteration and we + # slightly increase the learning rate L = L_next G, df, objective, total_active, a1, a2 = ( G_next, df_next, objective_next, total_active_next, a1_next, a2_next) From 9a61bc7829690c90873204041401b8e14b5f5be8 Mon Sep 17 00:00:00 2001 From: William de Vazelhes Date: Fri, 10 Aug 2018 11:09:57 +0200 Subject: [PATCH 09/11] FIX: remove non ascii character --- metric_learn/lmnn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metric_learn/lmnn.py b/metric_learn/lmnn.py index 19222574..f58bc00a 100644 --- a/metric_learn/lmnn.py +++ b/metric_learn/lmnn.py @@ -116,7 +116,7 @@ def fit(self, X, y): delta_obj = objective_next - objective if delta_obj > 0: # if we did not find a better objective, we retry with an L closer to - # the starting point, by decreasing the learning rate (making the + # the starting point, by decreasing the learning rate (making the # gradient step smaller) learn_rate /= 2 else: From 8b3f5848747a015159a276f7721db16e61aa013f Mon Sep 17 00:00:00 2001 From: William de Vazelhes Date: Sat, 18 Aug 2018 01:31:38 +0200 Subject: [PATCH 10/11] FIX: remove keyring.deb --- keyring.deb | Bin 3258 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 keyring.deb diff --git a/keyring.deb b/keyring.deb deleted file mode 100644 index cc48860b1cf8bc94f6c05644e7a630f337ada86e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3258 zcmai1c{J1w8$~51`&c5fCoRligko%2l6~LJgwc$lXl58OC?c;VLWn6!Y3yoL)*3IN z31i=tt;N2KZTNkC-}&D6eE)psz2`aioadf%pTF+?Ssjz3;6;rYQeW%+#cBIGIC*;e!?!6=S+zpWvJ+uBG{;!uKJBwtpI%oWA|~Tc zK@Bubxb;~7mcMu1y`SU~R`iyqYX|3s!5}<&da{CKh#w@UDOiYU-TFnkal|LAnG<~6 z55;lUn>SOHL-&<&IA-Yduw$?K>l<#Y3f77K3r8n_tyOZ`j0zx_C<(N6D^6paK7f`5 zQX8Wg)yqzg;uW@xzM0DEIM-)!z7@~JcI9$L`c|tm5254Wl{oJyLf|>}rrhLw`efA6 zIT&$r_s(24eNeZg3Zytar z?+N8eR7M+Ce*vH%|BO4{>|eVg&^PHCqSfgc@Drte{V~ntC+x-ST`}FYoX=Wmc(gzP zS5=c;SJ6anj;{EdMR#V$0A#ud)lNDqq)Xl9-bs}sW|OXf+W@G;{d1gd{uZ_&bzree zsRjZaTD-DX!V?xr9!K8$gE)HD$6I8kV86};5cA~5WDM=ktcAVfQRW6)J@IHAR#WXb zt=s{zkcRa0Q*HV=SI79IY^`LUhG&UtTOV%^$jNrKE!HPd*Bff_+?_=`(nRdSoIhVD z(75hxJng+kO4mL}ecCH#AbZgQ%+NF;Mq91!1ng#sx6iY+->rm}5Bb=7cVdNWpTcBX zYR%b&Xz$lXlzE5Z(G~IjY;leekrCxBq>z@Cz=8)?MeIL5u;ey1BiIWqgm2Hx4kzMP zuu71+HuDDhX~`WDy$MZT+wan4WoX=DT|AHiOsBj*AZWm~?bMgkldH?ya6aH3FOWJl zSs}HUffvkPvbG7MVi1L+=Z?l6anc zi)nt9aP%}!t%^*W%)QepVBF(ZI1B-e63!pldQO?33|82C?&E-QV2|5B!J5xDMLTns znz@eHqw;5%`+|Br4&>Y0=(AFXjW1tiN60~qxD2=uH!8QA=zC@RBs(tuKxps6v2}j1 zvWCzK9Csiti>y&?hhHu1js{9LRYf~gkK|pRx_g7`Tus+6a-bFG0h~gzeUx^Fbt+9t zWrkZTxb@%(Ng{5I%*rs+Sm$Tu(dO2+tq&i-wU63J+%L#fEtZ8Qd@GXN(;fP92gBN= zkfJ3$(YRarVzrWg^t}G&)tly9vGD%3*G}_byrtt@`WhBOT5J0}nD6%X5PuNpHaPy$ zL-h!quozLJ8YWCR#SxXt(%5*hk0)W z?vD&KEK>U2)rzx4PVCa8`Z4@N3*f^VIk{xsla#jw&$@I*aJg_J++HFoqR{P2$zw8N zKiyPWG^OI^ZDB7cMsY8tE)|8ThAC^K{VM$5otRTRABFKmPMc%cX(^4qGq(Hpja-`C zay|@P@pwq=`WTnR$DQ@ycm>I9#V(kJYzr?bZyl~Wq*l-fR3l^fG#i`j6lZO12P~(3 zUzG80%D1|q1o9uZ<2x?};$6^&{1xi)EyZsHFoa^U*i;xCrX^JcDz)_DM!PTyM=yK$?cFq6+EhnLYxVF(gyt`|47e7Y ze7(B)Jqhnw!RR*C^H)?^z|$vS*hJ0X{I5HPpSCCBDWRs_@?SjT%Ib)1_M#!)kJL)M zHjVf)sfuTw$7P%~?bJLk;U|Z+)7g-hj=L~3kL0zoyb?jmYSU zVD^l;-&jG~T+A&McV6WoRp!P<7v|Gt!tw}y&R=jzyW>H25B5g~_$|>be;Db(&~2;ol-)c_7AU#a zY+dF$DV-d)`i!}AgK)0t>aVCVj(dgALJh9}$nL0|)ckp-sao%&_a%d-I5ee{Vzldl z@J^QeZq0uNu02!cA7au2ul_Wldx}T=B@9a_d+w|AC8EK1ENU7fs`d_m7zArTk4j@h zbTD)L@*PDpG7IR>3z$-^q}8VKu;CHnFy=W=2AS~D)OpY<6Gt6R8yuuxpUH3x`GHXy z<)o2k&7HYUkz12L#kVm(7m2-FELR~u6&rPrsdqN)zBp^oa8fUg@8_Xxyje%HvkQ9AdH z%tW?TaG#ae-h#)I4^Iew-pv6;ULhhJ5tj(;j5zIToun9{xRf2qn$q?N$B7>2tduvZ z}Hi~J{U804wX zCG*Y>;Eutj4 zSY3DCaz)j_4j9w4BZk1l?g)EZTk@M)P94=Q)zS7I{M8sql*^y*gzFw_NSUgtWSS&U z!}_{uCfgl%UfOfh&q{dp!nUpG6UmqR#{^~*?eUFnD$|jufsJ1sy93yh8{ub&t2k0w zjF0Qhk*Ukc-|4+O$~<)QBAi%~;6M^Bv$&xgnqWY|KZ{?gt46;qT=kvN$=LKmSER&CTkFrTcoHQ>f+{+M;^0)SJw& z$7yPleJA9vMbWc~1sbm>xsxA6k@M3oyig?x_GO(3{NeE)DpjMUL`W|elW4~~nN9ie z>PF2rkK!Y0nS>Lt^Bi_uJoVx38yrp`cc-JO^^~xPUW|FhCfcgOx-dPMZg$_{p`JoY zrV1fZO{3$zo1MdX2_1#Y;vKCb;Jm(oM%VtX>SJ9lFVt1u7z@-$dEBbv)3ul;BL1Xh z_K2IGyfW6b*rKRJG;GX9jwWBI#QJ|>w&yG}wa~QL;A8xROcBHWjSWrVFK=$s;(5)( zOiCj(zaW*omSbC#f1Cx2jbC%YiQPLQUKYx85#KQK2u~#re0^<0VI{2`deb;fFG>xT z${!*tAqpSdn3rqfT6DD-y*nbmR2<&<-oay)X|ZeErOPO*$Gl#lt-nhwIlq454^W4r zK4Sv_19yyOr1-W{FE>sm0LDVevC-xlYJ#VFPeW!cSxR=wG6&X23Oow9(t`o&&-tlu zehRUYhY>-1-Kw>BxIlc-$D2wyJKBs7Ym8*>2j@aS4v9-L^FooFoi8d~(@5K+x@+pOl4Fm# z0#^E>a_51Dy43u`2PwPKaiB&3P6kq4Sw^gaU1_>xya9_0(% Date: Sat, 18 Aug 2018 01:36:08 +0200 Subject: [PATCH 11/11] STY: remove unused imports --- test/metric_learn_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/metric_learn_test.py b/test/metric_learn_test.py index 498fa816..1d0a5d02 100644 --- a/test/metric_learn_test.py +++ b/test/metric_learn_test.py @@ -1,7 +1,5 @@ -import re import unittest import re -import sys import pytest import numpy as np from scipy.optimize import check_grad