9
! #$% &'()*'+ ,)-'./0)12 &3-)45-*26+78 9 :;<& =:& >#? 5'()*'+@7)-'.+0)12A53-)B5-*2(+78@C- ,85'3+2 )C -+B-+2+'(5D)'2 !"# 5'E %& F578*'+ 1+5-'*'3 8GBH!!III@53-)B5-*2(+78@C-!.C-J*'C)!F+FK-+2!7)-'.+0)12!6+578*'3!=52(+-J&:,!=$J&:,J5EL5'7+EJ=M@8(F1 ! #$% N+5(.-+ 2+1+7D)' O*(8 -+3.15-*P+E 5BB-)Q*F5D)' $ R ,85'3+ )C -+B-+2+'(5D)' C)- 5'E KS F578*'+ 1+5-'*'3 T U&@ ,)-'./0)12V ! #$% W/1+7D)' E+ L5-*5K1+2 45- 5BB-)78+ N*1(+- O-5BB+- XFK+EE+E =5*2 E*Y7*1+ Z-5'E+ L5-*5'7+ E+2 -/2.1(5(2 +' C)'7D)' E+ 15 2/1+7D)' E+2 L5-*5K1+2 &BB-)78+ R !"#$%&'() T [' B/'51*2+ 1+2 L5-*5K1+2 +' C)'7D)' E+ 1+.- B)*E2 % R ,85'3+ )C -+B-+2+'(5D)' C)- 5'E KS F578*'+ 1+5-'*'3 T U&@ ,)-'./0)12V ! #$% WB5-2+ 1*'+5- -+3-+22*)' N-)F (8+ (.()-*51 \WB5-2+ F)E+1*'3@ 68+)-S] &13)-*(8F2 5'E &BB1*75D)'2^] *' :,=MJ$_#_ KS :-*'5 <*28 5'E Z+'5ES Z-5K5-'*` a R ,85'3+ )C -+B-+2+'(5D)' C)- 5'E KS F578*'+ 1+5-'*'3 T U&@ ,)-'./0)12V ! " ./ # %&'() *+,- ./0/ 12)%3&.'%456 7&8( 9 (/:;<)( 1=>??6 @&<A:%( 9 B&C)<( 1=D?E???6 F%G%&8% ;/7/:)0)7( 1H('4%/<56 +)/(A7):)%0(I :)%0/< (0/0)(E J)K/B'&7E 0/(G( &7 (0':A<' 0)*1 %2#33 *425&$ -6 2-%7 $&3&8#*7 8-/&3% 95$#)* #$&#%:

,85'3+2)C-+B-+2+'(5D)'2 - AgroParisTech … · m*'+5-"-+3-+22*)'h"(8+"b-)k1+f" • m+(b2"522.f+"i+"85l+"5"(-5*'*'3"2+("" • m*'+5-"-+3-+22*)'"1))`2"c)-"5"f)e+1")c"(8+"(sb+h" •

  • Upload
    others

  • View
    5

  • Download
    0

Embed Size (px)

Citation preview

Page 1: ,85'3+2)C-+B-+2+'(5D)'2 - AgroParisTech … · m*'+5-"-+3-+22*)'h"(8+"b-)k1+f" • m+(b2"522.f+"i+"85l+"5"(-5*'*'3"2+("" • m*'+5-"-+3-+22*)'"1))`2"c)-"5"f)e+1")c"(8+"(sb+h" •

!"#$%"

&'()*'+",)-'./0)12"

&3-)45-*26+78"9":;<&"""=:&">#?"

5'()*'+@7)-'.+0)12A53-)B5-*2(+78@C-"

,85'3+2")C"-+B-+2+'(5D)'2""!"#$5'E"%&$F578*'+"1+5-'*'3"

8GBH!!III@53-)B5-*2(+78@C-!.C-J*'C)!F+FK-+2!7)-'.+0)12!6+578*'3!=52(+-J&:,!=$J&:,J5EL5'7+EJ=M@8(F1"

!"#$%"

N+5(.-+"2+1+7D)'"

O*(8"-+3.15-*P+E"5BB-)Q*F5D)'"

$"R",85'3+")C"-+B-+2+'(5D)'"C)-"5'E"KS"F578*'+"1+5-'*'3"T""""""U&@",)-'./0)12V""

!"#$%"

W/1+7D)'"E+"L5-*5K1+2"

•  45-"5BB-)78+""–  N*1(+-"

–  O-5BB+-"

–  XFK+EE+E"

•  =5*2""

–  E*Y7*1+"

–  Z-5'E+"L5-*5'7+"E+2"-/2.1(5(2"+'"C)'7D)'"E+"15"2/1+7D)'"E+2"L5-*5K1+2"

"&BB-)78+"R"!"#$%&'()"T"

–  ['"B/'51*2+"1+2"L5-*5K1+2"+'"C)'7D)'"E+"1+.-"B)*E2"%"R",85'3+")C"-+B-+2+'(5D)'"C)-"5'E"KS"F578*'+"1+5-'*'3"T""""""U&@",)-'./0)12V"" !"#$%"

WB5-2+"1*'+5-"-+3-+22*)'"

"N-)F"(8+""(.()-*51"\WB5-2+"F)E+1*'3@"68+)-S]"&13)-*(8F2"5'E"&BB1*75D)'2 ]""

*'":,=MJ$_#_"KS":-*'5"<*28"5'E"Z+'5ES"Z-5K5-'*`"

a"R",85'3+")C"-+B-+2+'(5D)'"C)-"5'E"KS"F578*'+"1+5-'*'3"T""""""U&@",)-'./0)12V""

!"#$%&'()*&#$'+&,$&%%)-*''''

! "

./

#$$%&'()

*+,-

./0/$12)%3&.'%456

7&8($9

(/:;<)($1=>??6

@&<A:%($9

B&C)<(

1=D?E???6

F%G%&8%

;/7/:)0)7(

1H('4%/<56

+)/(A7):)%0(I

:)%0/<$(0/0)(E$J)K/B'&7E

0/(G($&7$(0':A<'

fMRI activation image and time-course courtesy of Steve Smith, FMRIB

0)*1'%2#33'*425&$'-6'2-%7'$&3&8#*7''8-/&3%

95$#)*'#$&#%:''''

Page 2: ,85'3+2)C-+B-+2+'(5D)'2 - AgroParisTech … · m*'+5-"-+3-+22*)'h"(8+"b-)k1+f" • m+(b2"522.f+"i+"85l+"5"(-5*'*'3"2+("" • m*'+5-"-+3-+22*)'"1))`2"c)-"5"f)e+1")c"(8+"(sb+h" •

!"#$%"

M*'+5-"-+3-+22*)'H"(8+"B-)K1+F"

•  M+(b2"522.F+"I+"85L+"5"(-5*'*'3"2+(""

•  M*'+5-"-+3-+22*)'"1))`2"C)-"5"F)E+1")C"(8+"(SB+H"

•  =)-+"3+'+-511S"UI8+'""""""""""""""""VH""

>"R",85'3+")C"-+B-+2+'(5D)'"C)-"5'E"KS"F578*'+"1+5-'*'3"T""""""U&@",)-'./0)12V""

S =!

(x1, y1), (x2, y2) . . . , (xm, ym)" x ! R

d

y ! R

y =

d!

k=1

wk x(k)

+ w0 U"<+3-+22*)'"V"

:'"(8+"C)11)I*'3]"C)-"-+52)'")C"2*FB1*7*(S]"I+"2D7`"()"(8+"752+")C"2*'31+"(5-3+("L5-*5K1+"!"

!"#$%"

M))`*'3"C)-"5"2B5-2+"F)E+1"

•  :E+511S]")'+"I).1E"1*`+"()".2+"52"C+I"52"B)22*K1+"L5-*5K1+2"QU0V"U512)"7511+E"K52*2"

C.'7D)'2"*C"(8+"*'B.("2B57+"*2"E+27-*K+E"KS"5"2+(")C"2.78"C.'7D)'2V"5'E"2D11"

B-+E*7("I+11"(8+"L51.+")C"!"

•  68*2"75'"K+"+'7).-53+E"KS".2*'3"5"B+'51(S")'"(8+"'.FK+-")C"B5-5F+(+-2H"

c"R",85'3+")C"-+B-+2+'(5D)'"C)-"5'E"KS"F578*'+"1+5-'*'3"T""""""U&@",)-'./0)12V""

<`H"O+"85L+"*'7)-B)-5(+E"I_"

*'"']"5'E"5EE+E")'+"

E*F+'2*)'"()"("*'")-E+-"()""

2*FB1*CS"(8+"')(5D)'@"

d.("(8*2"*2"')("+52S"()")BDF*P+"

!"#$%"

M))`*'3"C)-"5"2B5-2+"F)E+1H"-*E3+"-+3-+22*)'"

•  <*E3+"-+3-+22*)'"+'7).-53+"1)I"L51.+2")C"(8+"B5-5F+(+-2"

•  d.("*("E)+2"')("533-+22*L+1S"B.28"(8+F"()"P+-)"

e"R",85'3+")C"-+B-+2+'(5D)'"C)-"5'E"KS"F578*'+"1+5-'*'3"T""""""U&@",)-'./0)12V"" !"#$%"

M))`*'3"C)-"5"2B5-2+"F)E+1H"(8+"1522)"F+(8)E"

•  M522)"-+3-+22*)'"+'7).-53+"P+-)"L51.+2")C"(8+"B5-5F+(+-2"

–  :("E)+2")**#+,,-.+/&$01,2$32+$0)#)4+3+#,$3"$5+#"$

–  &'E"*2"-5(8+-"+),-/&$6"4013)%/+$

M&WW[H"M+52("&2)1.(+"W8-*'`53+"5'E"W+1+7D)'"[B+-5()-"f6*K28*-5'*]"#ggch"

?"R",85'3+")C"-+B-+2+'(5D)'"C)-"5'E"KS"F578*'+"1+5-'*'3"T""""""U&@",)-'./0)12V""

Page 3: ,85'3+2)C-+B-+2+'(5D)'2 - AgroParisTech … · m*'+5-"-+3-+22*)'h"(8+"b-)k1+f" • m+(b2"522.f+"i+"85l+"5"(-5*'*'3"2+("" • m*'+5-"-+3-+22*)'"1))`2"c)-"5"f)e+1")c"(8+"(sb+h" •

!"#$%"

:11.2(-5D)'"

"68+"*'(+-2+7D)'")C"(8+"7.-L+")C"+i.*J+--)-"I*(8"(8+"7)'2(-5*'("C.'7D)'"UI#"j"I$"k"#!!V"

U1522)]")'"(8+"1+lV")-""UI#"j"I$"k"#!!$V"U-*E3+]")'"(8+"-*38(V]"1+5E2"()"'5(.-511S"C5L)-"'.11"

7)+Y7*+'(2"*'"(8+"752+")C"(8+"1522)"F+(8)E"52"7)FB5-+E"()"(8+"-*E3+"-+3-+22*)'@"

g"R",85'3+")C"-+B-+2+'(5D)'"C)-"5'E"KS"F578*'+"1+5-'*'3"T""""""U&@",)-'./0)12V""

N-)F"fm52D+]"6*K28*-5'*]"N-*+EF5'@"\68+"+1+F+'(2")C"2(5D2D751"1+5-'*'3 @"WB-*'3+-]"$__gh]"B@e#"

!"#$%"

,8))2*'3"(8+"')-F"

•  ,)'L+Q*(S"" "no""+Y7*+'(")BDF*P5D)'"F+(8)E2"

•  45-7*F)'S""no""C+5(.-+"2+1+7D)'"

–  #"p"#"H""2B5-2*(S "K.("""')"7)'L+Q*(S"

–  $"o"#"H""7)'L+Q*(S""" "K.("""')"2B5-2*(S"

–  *$7$8"H""7)'L+Q*(S""""&;q""2B5-2*(S"

#_"R",85'3+")C"-+B-+2+'(5D)'"C)-"5'E"KS"F578*'+"1+5-'*'3"T""""""U&@",)-'./0)12V""

!"#$%"

M))`*'3"C)-"5"2B5-2+"F)E+1H"+152D7"'+("

•  :'"B-57D7+]"5"7)FK*'5D)'")C"1522)"5'E"-*E3+]"`')I'"52"+152D7"'+(]")l+'"

B+-C)-F2"K+2(]"2*'7+"*("B-)L*E+2"5"3))E"7)FK*'5D)'")C"2B5-2*(S"U1522)V"5'E"

-+3.15-*P5D)'"U-*E3+V@"

##"R",85'3+")C"-+B-+2+'(5D)'"C)-"5'E"KS"F578*'+"1+5-'*'3"T""""""U&@",)-'./0)12V""

!"#$%&'%()*+,

-&.&/0& 1

2&/&

*)334&'%()*+,

-&2&/0& 1

.&/

5*)3+"6&7%+

'%()*+,

!"#$%"

WB5-2*(S"-+3.15-*P5D)'"

•  68+"2+5-78"C)-")$*""9$2&0"32+,-,$,0)6+"H""U*@+@")C"5E+i.5(+"

75B57*(SV"*2"')I"28*l+E"()"(8+"2+5-78"C)-"32+$)00#"0#-)3+$

:14%+#$"!$%),-,$!1:6;":,"U)-"L5-*5K1+2V"

" " ",)'7+B(")C"#+*1/)#-5);":$0)32$

#$"R",85'3+")C"-+B-+2+'(5D)'"C)-"5'E"KS"F578*'+"1+5-'*'3"T""""""U&@",)-'./0)12V""

Page 4: ,85'3+2)C-+B-+2+'(5D)'2 - AgroParisTech … · m*'+5-"-+3-+22*)'h"(8+"b-)k1+f" • m+(b2"522.f+"i+"85l+"5"(-5*'*'3"2+("" • m*'+5-"-+3-+22*)'"1))`2"c)-"5"f)e+1")c"(8+"(sb+h" •

!"#$%"

<+3.15-*P5D)'"B5(8"

•  N-)F"fr+L*'"=.-B8S@"\=578*'+"M+5-'*'3@"&"4-)K5K*1*2D7"4+-2B+7DL+ @"=:6"4-+22]"$_#$h]"B@a%e"

#%"R",85'3+")C"-+B-+2+'(5D)'"C)-"5'E"KS"F578*'+"1+5-'*'3"T""""""U&@",)-'./0)12V""

13.3. !1 regularization: basics 437

0 5 10 15 20 25 30!0.2

!0.1

0

0.1

0.2

0.3

0.4

0.5

0.6

lcavol

lweight

age

lbph

svi

lcp

gleason

pgg45

(a)

0 5 10 15 20 25!0.2

!0.1

0

0.1

0.2

0.3

0.4

0.5

0.6

0.7

lcavol

lweight

age

lbph

svi

lcp

gleason

pgg45

(b)

Figure 13.7 (a) Profiles of ridge coe!cients for the prostate cancer example vs bound on !2 norm of w,so small t (large ") is on the left. The vertical line is the value chosen by 5-fold CV using the 1SE rule.Based on Figure 3.8 of (Hastie et al. 2009). Figure generated by ridgePathProstate. (b) Profiles of lassocoe!cients for the prostate cancer example vs bound on !1 norm of w, so small t (large ") is on the left.Based on Figure 3.10 of (Hastie et al. 2009). Figure generated by lassoPathProstate.

0 0.5 1 1.5 2!0.2

!0.1

0

0.1

0.2

0.3

0.4

0.5

0.6

0.7

!

lcavol

lweight

age

lbph

svi

lcp

gleason

pgg45

(a)

1 2 3 4 5 6 7 8 9!0.2

!0.1

0

0.1

0.2

0.3

0.4

0.5

0.6

0.7

lars step

lcavol

lweight

age

lbph

svi

lcp

gleason

pgg45

(b)

Figure 13.8 Illustration of piecewise linearity of regularization path for lasso on the prostate cancerexample. (a) We plot wj(B) vs B for the critical values of B. (b) We plot vs steps of the LARS algorithm.Figure generated by lassoPathProstate.

B, the coe!cients gradually “turn on”. But for any value between 0 and Bmax = ||wOLS ||1,the solution is sparse.4

Remarkably, it can be shown that the solution path is a piecewise linear function of B (Efronet al. 2004). That is, there are a set of critical values of B where the active set of non-zerocoe!cients changes. For values of B between these critical values, each non-zero coe!cientincreases or decreases in a linear fashion. This is illustrated in Figure 13.8(a). Furthermore,one can solve for these critical values analytically. This is the basis of the LARS algorithm(Efron et al. 2004), which stands for “least angle regression and shrinkage” (see Section 13.4.2for details). Remarkably, LARS can compute the entire regularization path for roughly the same

4. It is common to plot the solution versus the shrinkage factor, defined as s(B) = B/Bmax , rather than against B.This merely a"ects the scale of the horizontal axis, not the shape of the curves.

!"#$%"

,8+F*'"E+"-/3.15D)'"

X'2+FK1+"E+2"B)*E2"522)7*/2"5.Q"E*s/-+'(2"

C57(+.-2"B).-".'"0+."E+"E)''/+2"5-Dt7*+1"

1)-2i.+"1u)'"C5*("L5-*+-"15"2)FF+"E+2"L51+.-2"

5K2)1.+2"E+2"B)*E2"E+2"C57(+.-2"E+"_"v"15"

2)FF+"F5Q*F51+"B)22*K1+"E5'2".'+"B-)7/E.-+"

M&<@""

<:$2)13"t3.-+"1+"')FK-+"E+"C57(+.-2"B-*2"+'"

7)FB(+"UE+"_"v"#_V@""

=$*)162+]".'"7)+Y7*+'("')-F51*2/"B).-"1+2"

C57(+.-2@""

=$9#"-3+]"1+2"'.F/-)2"E+2"C57(+.-2@""

['"L)*("i.+"5."C.-"+("v"F+2.-+"i.+"1u)'"

5.3F+'(+"1+"B5-5Fw(-+"E+"7)'(-x1+]"E+"B1.2"+'"

B1.2"E+"C57(+.-2"B-+''+'(".'"B)*E2"')'"'.1"

UD-/"Eu.'"(.()-*+1"E+"6-+L)-"m52D+"+'"2+B(@"

$__cV"

#a"R",85'3+")C"-+B-+2+'(5D)'"C)-"5'E"KS"F578*'+"1+5-'*'3"T""""""U&@",)-'./0)12V""

!"#$%"

&13)-*(8F2"

•  "W(5'E5-E"i.5E-5D7"B-)3-5FF*'3"F+(8)E2H"())"21)I"

•  "M+52("&'31+"<+3-+22*)'"UM&<WV"fXC-)'"+("51@]"$__ahH"

–  =.78"C52(+-"

–  4-)E.7+2"(8+"+'D-+"2)1.D)'"B5(8"5("(8+"7)2(")C"5"2*'31+"1+52(J2i.5-+2"t("

•  ",))-E*'5(+"E+27+'(""

–  fN.]"#gg?h]"fq5.K+78*+2"+("51@]"$__ah]"fN-*+EF5'"+("51@]"$__eh]"fO."y"M5'3+]"$__?h"

–  N)-"tQ+E"!]")BDF*P+2"+578"B5-5F+(+-"5("5"DF+"

–  :("75'"7)FB.(+"(8+"2)1.D)'2")'"5"3-*E")C"!"L51.+2"C52(+-"(85'"M&<W"U8)I+L+-]"(8+"C.11"B5(8"*2";[6"7)FB.(+EV"

•  "=5'S")(8+-"F+(8)E2]"*'71.E*'3"3+'+-51*P5D)'2"()")(8+-"1)22+2"

–  z5-*).2"B57`53+2]"+@3@"2++"8GBH!!E2B@-*7+@+E.!72"

#>"R",85'3+")C"-+B-+2+'(5D)'"C)-"5'E"KS"F578*'+"1+5-'*'3"T""""""U&@",)-'./0)12V"" !"#$%"

6+78'*i.+2"+'"B1+*'"E/L+1)BB+F+'("

•  {(.E+2"(8/)-*i.+2"

•  <+78+-78+"Eb+'2+FK1+2"E+"L5-*5K1+2"2(-.7(.-/2"

–  Z-).B"M&WW["

–  |"

#c"R",85'3+")C"-+B-+2+'(5D)'"C)-"5'E"KS"F578*'+"1+5-'*'3"T""""""U&@",)-'./0)12V""

Page 5: ,85'3+2)C-+B-+2+'(5D)'2 - AgroParisTech … · m*'+5-"-+3-+22*)'h"(8+"b-)k1+f" • m+(b2"522.f+"i+"85l+"5"(-5*'*'3"2+("" • m*'+5-"-+3-+22*)'"1))`2"c)-"5"f)e+1")c"(8+"(sb+h" •

!"#$%"

WB5-2+"7)E*'3"

5'E")L+-7)FB1+(+"K52+2"

#e"R",85'3+")C"-+B-+2+'(5D)'"C)-"5'E"KS"F578*'+"1+5-'*'3"T""""""U&@",)-'./0)12V"" !"#$%"

WB5-2+"7)E*'3"

•  O+"522.F+"(85("I+"I5'("()"-+7)L+-""""""""""""""""""C-)F"5"

F+52.-+F+'(""

•  O+"2.BB)2+"(8*2"*2"B)22*K1+"(8-).38"5"E*7D)'5-S">"U)-"?V"

#?"R",85'3+")C"-+B-+2+'(5D)'"C)-"5'E"KS"F578*'+"1+5-'*'3"T""""""U&@",)-'./0)12V""

x ! Rd

[L+-J7)FB1+(+"K52*2"

!"#$%"

}2+")C"\K52*2"C.'7D)'2^"

•  [l+']"(8+"*'B.("2B57+"*2"(-5'2C)-F+E"*'()"5"-+E+27-*BD)'"2B57+"

I8)2+"%&"E*F+'2*)'2"5-+"\K52*2"C.'7D)'2^"U+@3@"I5L+1+(2V"

•  O8+'"(8+"E*F+'2*)'"%&")C"(8+"-+E+27-*BD)'"2B57+"*2"C5-"3-+5(+-"(85'"(8+"*'B.("

2B57+"E*F+'2*)'"%]"(8+"2B57+")C"K52*2"C.'7D)'2"*2"7511+E"5'"".+#6"40/+3+$%),-,"U)-"5"9-6;":)#&V"

–  :'2B*-+E"C-)F"(8+"'+.-)K*)1)3S")C"(8+"L*2.51"2S2(+F"f[1285.2+'"y"N*+1E]"#ggch"

–  N+I"\I)-E2^"C-)F"5"15-3+"E*7D)'5-S"5-+"+').38"()"E+27-*K+"5'S"*'B.("

#g"R",85'3+")C"-+B-+2+'(5D)'"C)-"5'E"KS"F578*'+"1+5-'*'3"T""""""U&@",)-'./0)12V""

x ! Rd@:,10+#.-,+9$$

,+A:*$

!"#$%"

:11.2(-5D)'")'"\E+')*2*'3"*F53+2^"

N-)F"f6.()-*51"X<=:6XWJ$_#_"KS"~.1*+'"=5*-51"\WB5-2+"7)E*'3"5'E"E*7D)'5-S"1+5-'*'3"C)-"*F53+"5'51S2*2^h"

$_"R",85'3+")C"-+B-+2+'(5D)'"C)-"5'E"KS"F578*'+"1+5-'*'3"T""""""U&@",)-'./0)12V""

The Image Denoising Problem

y!"#$

measurements

= xorig!"#$

original image

+ w!"#$noise

Julien Mairal Sparse Coding and Dictionary Learning 6/182

Page 6: ,85'3+2)C-+B-+2+'(5D)'2 - AgroParisTech … · m*'+5-"-+3-+22*)'h"(8+"b-)k1+f" • m+(b2"522.f+"i+"85l+"5"(-5*'*'3"2+("" • m*'+5-"-+3-+22*)'"1))`2"c)-"5"f)e+1")c"(8+"(sb+h" •

!"#$%"

q+')*2*'3"*'B.(2"

68+"5BB-)578"

•  X'+-3S"F*'*F*P5D)'"B-)K1+F"pJo"=5Q*F.F"&"4)2(+-*)-*"+2DF5D)'"

$#"R",85'3+")C"-+B-+2+'(5D)'"C)-"5'E"KS"F578*'+"1+5-'*'3"T""""""U&@",)-'./0)12V""

m+-+]"(8+"0#-"#$*2"(85(")-*3*'51$-4)*+,$6):$%+$9+,6#-%+9$1,-:*$!+'$'"#9,$-:$)$9-6;":)#&$

!"#$%"

q*7D)'5-*+2"

•  6+FB)-51"2*3'51"

•  :F53+2""

$$"R",85'3+")C"-+B-+2+'(5D)'"C)-"5'E"KS"F578*'+"1+5-'*'3"T""""""U&@",)-'./0)12V""

What is a Sparse Linear Model?

Let x in Rm be a signal.

Let D = [d1, . . . ,dp] ! Rm!p be a set ofnormalized “basis vectors”.We call it dictionary.

D is “adapted” to y if it can represent it with a few basis vectors—thatis, there exists a sparse vector ! in Rp such that y " D!. We call !the sparse code.

!

"y

#

$

% &' (

y!Rm

"

!

" d1 d2 · · · dp

#

$

% &' (

D!Rm!p

!

)))"

![1]![2]...

![p]

#

***$

% &' (

!!Rp,sparse

Julien Mairal Sparse Coding and Dictionary Learning 8/182

What is a Sparse Linear Model?

Let x in Rm be a signal.

Let D = [d1, . . . ,dp] ! Rm!p be a set ofnormalized “basis vectors”.We call it dictionary.

D is “adapted” to y if it can represent it with a few basis vectors—thatis, there exists a sparse vector ! in Rp such that y " D!. We call !the sparse code.

!

"y

#

$

% &' (

y!Rm

"

!

" d1 d2 · · · dp

#

$

% &' (

D!Rm!p

!

)))"

![1]![2]...

![p]

#

***$

% &' (

!!Rp,sparse

Julien Mairal Sparse Coding and Dictionary Learning 8/182

4-+JE+(+-F*'+E"E*7D)'5-S"

U+@3@"N).-*+-"2+-*+2V"

What is a Sparse Linear Model?

Let x in Rm be a signal.

Let D = [d1, . . . ,dp] ! Rm!p be a set ofnormalized “basis vectors”.We call it dictionary.

D is “adapted” to y if it can represent it with a few basis vectors—thatis, there exists a sparse vector ! in Rp such that y " D!. We call !the sparse code.

!

"y

#

$

% &' (

y!Rm

"

!

" d1 d2 · · · dp

#

$

% &' (

D!Rm!p

!

)))"

![1]![2]...

![p]

#

***$

% &' (

!!Rp,sparse

Julien Mairal Sparse Coding and Dictionary Learning 8/182

What is a Sparse Linear Model?

Let x in Rm be a signal.

Let D = [d1, . . . ,dp] ! Rm!p be a set ofnormalized “basis vectors”.We call it dictionary.

D is “adapted” to y if it can represent it with a few basis vectors—thatis, there exists a sparse vector ! in Rp such that y " D!. We call !the sparse code.

!

"y

#

$

% &' (

y!Rm

"

!

" d1 d2 · · · dp

#

$

% &' (

D!Rm!p

!

)))"

![1]![2]...

![p]

#

***$

% &' (

!!Rp,sparse

Julien Mairal Sparse Coding and Dictionary Learning 8/182

M+5-'+E"E*7D)'5-S"

U)L+-"*F53+"B5(78+2V"

!"#$%"

q*7D)'5-S"1+5-'*'3"

$%"R",85'3+")C"-+B-+2+'(5D)'"C)-"5'E"KS"F578*'+"1+5-'*'3"T""""""U&@",)-'./0)12V"" !"#$%"

q*7D)'5-*+2"

•  q*7D)'5-*+2"75'"K+"9+,-*:+9$'+,#$-#$$$–  X@3@"O5L+1+(2]",.-L+1+(2]"O+E3+1+(2]"d5'E1+(2]"|"1+(2"

–  W++"5"L52("1*(+-5(.-+")'"*F53+"5'51S2*2"

•  q*7D)'5-*+2"75'"K+"/+)#:+9$

$a"R",85'3+")C"-+B-+2+'(5D)'"C)-"5'E"KS"F578*'+"1+5-'*'3"T""""""U&@",)-'./0)12V""

Page 7: ,85'3+2)C-+B-+2+'(5D)'2 - AgroParisTech … · m*'+5-"-+3-+22*)'h"(8+"b-)k1+f" • m+(b2"522.f+"i+"85l+"5"(-5*'*'3"2+("" • m*'+5-"-+3-+22*)'"1))`2"c)-"5"f)e+1")c"(8+"(sb+h" •

!"#$%"

M+5-'+E"E*7D)'5-S"C)-"*F53+"-+2()-5D)'"

W++"rJWzq"fX15E"y"&85-)']"$__ch"

$>"R",85'3+")C"-+B-+2+'(5D)'"C)-"5'E"KS"F578*'+"1+5-'*'3"T""""""U&@",)-'./0)12V""

Sparse representations for image restorationK-SVD: [Elad and Aharon, 2006]

Figure: Dictionary trained on a noisy version of the imageboat.

Julien Mairal Sparse Coding and Dictionary Learning 13/182

q*7D)'5-S"(-5*'+E")'"5"')*2S"L+-2*)'")C"(8+"*F53+")C"(8+"K)5("U)L+-"B5(78"*F53+2V"

!"#$%"

M+5-'+E"E*7D)'5-S"C)-"*F53+"-+2()-5D)'"

W++"rJWzq"fX15E"y"&85-)']"$__ch"

$c"R",85'3+")C"-+B-+2+'(5D)'"C)-"5'E"KS"F578*'+"1+5-'*'3"T""""""U&@",)-'./0)12V""

Z-5S2751+"L2@"7)1)-"*F53+"B5(78+2"

Sparse representations for image restorationGrayscale vs color image patches

Julien Mairal Sparse Coding and Dictionary Learning 14/182

!"#$%"

q*7D)'5-S"1+5-'*'3"

•  B":$6":.+($"0;4-5);":$0#"%/+4"K+75.2+"K*1*'+5-"*'"(I)")K0+7DL+2"

•  ,1522*751")BDF*P5D)'"51(+-'5(+2"K+(I++'"1+5-'*'3"?"5'E""#

#@  "C+)#62$!"#$)$,0)#,+$6"9-:*"""""3*L+'"5"E*7D)'5-S"?$

$@  "C+)#62$!"#$)$9-6;":)#&"?""""""""""3*L+'"(8+"7)E+""""

–  Z))E"-+2.1(2"K.("21)I�"

–  &')(8+-"51(+-'5DL+"*2"()".2+"":/-:+$/+)#:-:*"f=5*-51"+("51@]"$__gh"

$e"R",85'3+")C"-+B-+2+'(5D)'"C)-"5'E"KS"F578*'+"1+5-'*'3"T""""""U&@",)-'./0)12V"" !"#$%"

,)FB5-*2)'")C"7)FB)'+'(2"1+5-'*'3"

•  ['"5"C57+"5'51S2*2"B-)K1+F"

$?"R",85'3+")C"-+B-+2+'(5D)'"C)-"5'E"KS"F578*'+"1+5-'*'3"T""""""U&@",)-'./0)12V""

Matrix Factorization Problems and Dictionary LearningFaces

(d) PCA (e) NNMF (f) DL

Julien Mairal Sparse Coding and Dictionary Learning 73/182

Page 8: ,85'3+2)C-+B-+2+'(5D)'2 - AgroParisTech … · m*'+5-"-+3-+22*)'h"(8+"b-)k1+f" • m+(b2"522.f+"i+"85l+"5"(-5*'*'3"2+("" • m*'+5-"-+3-+22*)'"1))`2"c)-"5"f)e+1")c"(8+"(sb+h" •

!"#$%"

&BB-+'D2253+"E+"E*7D)''5*-+"

$g"R",85'3+")C"-+B-+2+'(5D)'"C)-"5'E"KS"F578*'+"1+5-'*'3"T""""""U&@",)-'./0)12V""

'"+Q+FB1+2" E*7D)''5*-+"

!"#$%"

q*7D)'5-S"1+5-'*'3"

*'"5"2.B+-L*2+E"7)'(+Q(Ä"

%_"R",85'3+")C"-+B-+2+'(5D)'"C)-"5'E"KS"F578*'+"1+5-'*'3"T""""""U&@",)-'./0)12V""

!"#$%"

q*7D)'5-S"1+5-'*'3"C)-"5"2.B+-L*2+E"(52`"

•  O+"I5'("

O*(8"(8+"(-5*'*'3"2+(""

•  D+$),,14+""(8+-+"+Q*2(2"5"2B5-2+"-+B-+2+'(5D)'")C"("*'"2)F+"E*7D)'5-S"

%#"R",85'3+")C"-+B-+2+'(5D)'"C)-"5'E"KS"F578*'+"1+5-'*'3"T""""""U&@",)-'./0)12V""

M+5-'+E"E*7D)'5-S""

C-)F"(8+"(-5*'*'3"2+("M+5-'*'3"B-)K1+F"

!"#$%"

q*7D)'5-S"1+5-'*'3"C)-"5"2.B+-L*2+E"(52`"

•  Z+'+-511S"2+B5-5(+E"*'"(I)"B852+2"

#@  "N*'E"5"E*7D)'5-S"-+B-+2+'(5D)'"*'"5'"1:,10+#.-,+9$2+Å'3"

$@  "}2+"(8*2"E*7D)'5-S"*'"(8+",10+#.-,+9$(52`"

%$"R",85'3+")C"-+B-+2+'(5D)'"C)-"5'E"KS"F578*'+"1+5-'*'3"T""""""U&@",)-'./0)12V""

Page 9: ,85'3+2)C-+B-+2+'(5D)'2 - AgroParisTech … · m*'+5-"-+3-+22*)'h"(8+"b-)k1+f" • m+(b2"522.f+"i+"85l+"5"(-5*'*'3"2+("" • m*'+5-"-+3-+22*)'"1))`2"c)-"5"f)e+1")c"(8+"(sb+h" •

!"#$%"

N":"W":","&"

&'"+QB+-*F+'("*'"2.B+-L*2+E"*F53+"-+7)3'*D)'"

%%"R",85'3+")C"-+B-+2+'(5D)'"C)-"5'E"KS"F578*'+"1+5-'*'3"T""""""U&@",)-'./0)12V"" !"#$%"

<+C+-+'7+2"

"   &@",)-'./0)12"y"M@"=*71+("

&BB-+'D2253+"&-Dt7*+1@",)'7+B(2"+("513)-*(8F+2"

XS-)11+2"U$Ç"/E@V]"$_#_"

"   É@"d+'3*)]"&@",).-L*11+"y"4@"z*'7+'("

<+B-+2+'(5D)'"M+5-'*'3H"&"<+L*+I"5'E";+I"4+-2B+7DL+2"

5-Ñ*LH#$_c@>>%?L%]"&B-*1"$_#a"

"   ~@"M++"y"=@"z+-1+S2+'"

;)'1*'+5-"E*F+'2*)'51*(S"-+E.7D)'"

WB-*'3+-]"$__e"

"   ~.1*+'"=5*-51"

WB5-2+"7)E*'3"5'E"E*7D)'5-S"1+5-'*'3"C)-"*F53+"5'51S2*2"

6.()-*51"21*E+2"C)-"X<=:6XW"$_#_"U8GBH!!1+5-@*'-*51B+2@C-!B+)B1+!F5*-51!-+2).-7+2!BEC!

X<=:6XW#_@BECV"

%a"R",85'3+")C"-+B-+2+'(5D)'"C)-"5'E"KS"F578*'+"1+5-'*'3"T""""""U&@",)-'./0)12V""