%PDF-1.3 1 0 obj << /Kids [ 3 0 R 4 0 R 5 0 R 6 0 R 7 0 R 8 0 R 9 0 R 10 0 R 11 0 R ] /Type /Pages /Count 9 >> endobj 2 0 obj << /Title (Delving Deep into Rectifiers\072 Surpassing Human\055Level Performance on ImageNet Classification) /Producer (PyPDF2) /Author (Kaiming He\054 Xiangyu Zhang\054 Shaoqing Ren\054 Jian Sun) /Subject (2015 IEEE International Conference on Computer Vision) >> endobj 3 0 obj << /Parent 1 0 R /Rotate 0 /TrimBox [ 0 0 612 792 ] /Contents 13 0 R /Resources << /XObject << /x8 14 0 R /x6 17 0 R /x12 20 0 R /x10 23 0 R >> /ExtGState << /s9 26 0 R /R9 29 0 R /s5 32 0 R /s7 35 0 R /s11 38 0 R /a0 << /CA 1 /ca 1 >> /R23 41 0 R /R22 42 0 R >> /Font << /F2 43 0 R /F1 44 0 R /R14 45 0 R /R16 49 0 R /R18 54 0 R /R20 59 0 R >> /ProcSet [ /ImageC /Text /PDF /ImageI /ImageB ] /ColorSpace << /R11 64 0 R /R13 66 0 R >> >> /MediaBox [ 0 0 612 792 ] /Annots [ ] /Type /Page /BleedBox [ 0 0 612 792 ] /ArtBox [ 0 0 612 792 ] >> endobj 4 0 obj << /Parent 1 0 R /Rotate 0 /TrimBox [ 0 0 612 792 ] /Contents 68 0 R /Resources << /ColorSpace << /R11 64 0 R /R13 66 0 R >> /ExtGState << /R23 41 0 R /R22 42 0 R /R9 29 0 R >> /Font << /R58 69 0 R /R16 49 0 R /R48 73 0 R /R54 78 0 R /R46 83 0 R /R56 88 0 R /R44 92 0 R /R50 97 0 R /R42 102 0 R /R52 106 0 R /R40 110 0 R /F2 114 0 R /R36 115 0 R /F1 120 0 R /R14 45 0 R /R38 121 0 R /R18 54 0 R /R34 126 0 R /R32 131 0 R /R30 135 0 R >> /ProcSet [ /ImageC /Text /PDF /ImageI /ImageB ] >> /MediaBox [ 0 0 612 792 ] /Annots [ ] /Type /Page /BleedBox [ 0 0 612 792 ] /ArtBox [ 0 0 612 792 ] >> endobj 5 0 obj << /Parent 1 0 R /Rotate 0 /TrimBox [ 0 0 612 792 ] /Contents 138 0 R /Resources << /ColorSpace << /R11 64 0 R /R13 66 0 R >> /ExtGState << /R23 41 0 R /R66 139 0 R /R87 140 0 R >> /Font << /R90 141 0 R /R79 145 0 R /R16 49 0 R /R48 73 0 R /R54 78 0 R /R73 149 0 R /R56 88 0 R /R44 92 0 R /R50 97 0 R /R42 102 0 R /R77 154 0 R /F2 158 0 R /R18 54 0 R /R88 159 0 R /R83 163 0 R /R81 168 0 R /R75 173 0 R /R92 177 0 R /R85 181 0 R /R71 185 0 R /R67 189 0 R /F1 194 0 R /R14 45 0 R /R38 121 0 R /R36 115 0 R /R34 126 0 R /R69 195 0 R >> /ProcSet [ /ImageC /Text /PDF /ImageI /ImageB ] >> /MediaBox [ 0 0 612 792 ] /Annots [ ] /Type /Page /BleedBox [ 0 0 612 792 ] /ArtBox [ 0 0 612 792 ] >> endobj 6 0 obj << /Parent 1 0 R /Rotate 0 /TrimBox [ 0 0 612 792 ] /Contents 199 0 R /Resources << /ColorSpace << /R13 66 0 R >> /ExtGState << /R23 41 0 R /R22 42 0 R /R9 29 0 R >> /Font << /R58 69 0 R /R16 49 0 R /R48 73 0 R /R54 78 0 R /R46 83 0 R /R56 88 0 R /R44 92 0 R /R50 97 0 R /R42 102 0 R /R40 110 0 R /F2 200 0 R /R18 54 0 R /F1 201 0 R /R83 163 0 R /R75 173 0 R /R77 154 0 R /R100 202 0 R /R14 45 0 R /R38 121 0 R /R36 115 0 R /R34 126 0 R /R69 195 0 R >> /ProcSet [ /ImageC /Text /PDF /ImageI /ImageB ] >> /MediaBox [ 0 0 612 792 ] /Annots [ ] /Type /Page /BleedBox [ 0 0 612 792 ] /ArtBox [ 0 0 612 792 ] >> endobj 7 0 obj << /Parent 1 0 R /Rotate 0 /TrimBox [ 0 0 612 792 ] /Contents 205 0 R /Resources << /ColorSpace << /R11 64 0 R /R13 66 0 R >> /ExtGState << /R23 41 0 R /R22 42 0 R /R9 29 0 R >> /Font << /R58 69 0 R /R48 73 0 R /R54 78 0 R /R46 83 0 R /R56 88 0 R /R44 92 0 R /R50 97 0 R /R52 106 0 R /F2 206 0 R /R88 159 0 R /R108 207 0 R /R112 210 0 R /R110 213 0 R /R114 218 0 R /F1 223 0 R /R14 45 0 R /R16 49 0 R /R18 54 0 R /R20 59 0 R >> /ProcSet [ /ImageC /Text /PDF /ImageI /ImageB ] >> /MediaBox [ 0 0 612 792 ] /Annots [ ] /Type /Page /BleedBox [ 0 0 612 792 ] /ArtBox [ 0 0 612 792 ] >> endobj 8 0 obj << /Parent 1 0 R /Rotate 0 /TrimBox [ 0 0 612 792 ] /Contents 224 0 R /Resources << /ColorSpace << /R13 66 0 R >> /ExtGState << /R23 41 0 R /R66 139 0 R >> /Font << /R48 73 0 R /R44 92 0 R /R50 97 0 R /R42 102 0 R /R18 54 0 R /F2 225 0 R /R36 115 0 R /F1 226 0 R /R14 45 0 R /R122 227 0 R /R16 49 0 R /R83 163 0 R /R34 126 0 R /R69 195 0 R >> /ProcSet [ /ImageC /Text /PDF /ImageI /ImageB ] >> /MediaBox [ 0 0 612 792 ] /Annots [ ] /Type /Page /BleedBox [ 0 0 612 792 ] /ArtBox [ 0 0 612 792 ] >> endobj 9 0 obj << /Parent 1 0 R /Rotate 0 /TrimBox [ 0 0 612 792 ] /Contents 230 0 R /Resources << /ColorSpace << /R11 64 0 R /R13 66 0 R >> /ExtGState << /R23 41 0 R /R66 139 0 R /R87 140 0 R >> /Font << /F2 231 0 R /R16 49 0 R /R73 149 0 R /R50 97 0 R /R130 232 0 R /R18 54 0 R /R132 236 0 R /F1 240 0 R /R134 241 0 R /R136 245 0 R /R138 249 0 R /R14 45 0 R >> /ProcSet [ /ImageC /Text /PDF /ImageI /ImageB ] >> /MediaBox [ 0 0 612 792 ] /Annots [ ] /Type /Page /BleedBox [ 0 0 612 792 ] /ArtBox [ 0 0 612 792 ] >> endobj 10 0 obj << /Parent 1 0 R /Rotate 0 /TrimBox [ 0 0 612 792 ] /Contents 253 0 R /Resources << /XObject << /R156 254 0 R /R157 255 0 R /R154 256 0 R /R155 257 0 R /R152 258 0 R /R153 259 0 R /R147 260 0 R /R158 261 0 R /R159 262 0 R /R160 263 0 R /R146 264 0 R >> /ExtGState << /R23 41 0 R /R66 139 0 R /R87 140 0 R /R22 42 0 R >> /Font << /F2 265 0 R /R18 54 0 R /F1 266 0 R /R14 45 0 R /R150 267 0 R /R16 49 0 R /R83 163 0 R /R148 272 0 R >> /ProcSet [ /Text /ImageC /ImageB /PDF /ImageI ] /ColorSpace << /R11 64 0 R /R13 66 0 R >> >> /MediaBox [ 0 0 612 792 ] /Annots [ ] /Type /Page /BleedBox [ 0 0 612 792 ] /ArtBox [ 0 0 612 792 ] >> endobj 11 0 obj << /Parent 1 0 R /Rotate 0 /TrimBox [ 0 0 612 792 ] /Contents 277 0 R /Resources << /ColorSpace << /R13 66 0 R >> /ExtGState << /R23 41 0 R /R9 29 0 R >> /Font << /R18 54 0 R /F2 278 0 R /F1 279 0 R /R16 49 0 R >> /ProcSet [ /ImageC /Text /PDF /ImageI /ImageB ] >> /MediaBox [ 0 0 612 792 ] /Annots [ ] /Type /Page /BleedBox [ 0 0 612 792 ] /ArtBox [ 0 0 612 792 ] >> endobj 12 0 obj << /Type /Catalog /Pages 1 0 R >> endobj 13 0 obj << /Length 18635 >> stream q q q 0.10000 0 0 0.10000 0 0 cm /R9 gs /R13 cs 1 scn q 10 0 0 10 0 0 cm BT /R14 14.34620 Tf 1 0 0 1 209.75700 675.06700 Tm [ (Delving) -249.99100 (Deep) -250.00800 (into) -249.99300 (Recti\223ers\072) ] TJ -114.99800 -17.93280 Td [ (Sur) 9.99540 (passing) -249.98900 (Human\055Le) 14.98930 (v) 9.99625 (el) -249.98800 (P) 20.00610 (erf) 24.98890 (ormance) -250 (on) -250.01300 (ImageNet) -250.01500 (Classi\223cation) ] TJ /R16 11.95520 Tf 41.63160 -37.85820 Td [ (Kaiming) -250.10100 (He) -2250.01000 (Xiangyu) -250 (Zhang) -2250.02000 (Shaoqing) -249.99300 (Ren) -2250 (Jian) -249.98500 (Sun) ] TJ 114.24400 -15.34220 Td [ (Microsoft) -250.01000 (Research) ] TJ /R14 11.95520 Tf -104.64000 -41.46520 Td (Abstract) Tj /R18 9.96260 Tf -83.92770 -13.94690 Td [ (Recti\223ed) -489.00100 (activation) -488.89800 (units) -488.00600 (\050r) 37.01590 (ecti\223er) 10.01060 (s\051) -489.01400 (ar) 36.98650 (e) -488.98200 (essential) -488.98200 (for) ] TJ -11.95510 -11.95630 Td [ (state\055of\055the\055art) -427.01600 (neur) 14.99010 (al) -427.99500 (networks\056) -842.01900 (In) -428.00600 (this) -427.00200 (work\054) -472.01700 (we) -427.99700 (study) ] TJ 11.95510 TL T* [ (r) 37.01960 (ecti\223er) -316.99400 (neur) 14.99010 (al) -316.99400 (networks) -317.01100 (for) -316.98800 (ima) 10.01300 (g) 10.00320 (e) -317.99100 (classi\223cation) -318.00800 (fr) 44.98510 (om) -316.99900 (two) ] TJ T* [ (aspects\056) -685 (F) 45.01580 (ir) 10.00450 (st\054) -405.99500 (we) -374.98800 (pr) 44.98390 (opose) -375.01800 (a) -374.99100 (P) 79.99030 (ar) 14.98890 (ametric) -375.01600 (Recti\223ed) -374.98200 (Linear) ] TJ T* [ (Unit) -244.01600 (\050PReLU\051) -243.90500 (that) -244.01300 (g) 10.00320 (ener) 15.01960 (alizes) -244.01100 (the) -243.99100 (tr) 14.99140 (aditional) -243.98400 (r) 37.01830 (ecti\223ed) -243.98600 (unit\056) ] TJ T* [ (PReLU) -300.98800 (impr) 44.99370 (o) 10.00320 (ves) -301.01100 (model) -300.01200 (\223tting) -301.01100 (with) -301.01500 (nearly) -300.99400 (zer) 44.98020 (o) -301.00400 (e) 19.99180 (xtr) 14.98770 (a) -301.00400 (com\055) ] TJ T* [ (putational) -249.89900 (cost) -248.88600 (and) -249.98300 (little) -250.00500 (o) 10.00320 (ver\223tting) -248.98300 (risk\056) -310.01000 (Second\054) -249.98300 (we) -249.98800 (derive) ] TJ T* [ (a) -418.00400 (r) 45.01820 (ob) 20.00650 (ust) -417.99000 (initialization) -417.99900 (method) -417.98800 (that) -419.00400 (particularly) -417.99600 (consider) 9.98363 (s) ] TJ T* [ (the) -267.98700 (r) 37.01960 (ecti\223er) -267.98200 (nonlinearities\056) -363.99000 (This) -268 (method) -266.99500 (enables) -268.01400 (us) -267.98200 (to) -267.98200 (tr) 14.99010 (ain) ] TJ T* [ (e) 19.99240 (xtr) 36.98340 (emely) -234.00300 (deep) -233.01900 (r) 37.01960 (ecti\223ed) -233.98900 (models) -232.99800 (dir) 36.99260 (ectly) -233.99800 (fr) 44.98510 (om) -233.01300 (scr) 14.98520 (atc) 14.98520 (h) -233.99500 (and) -233.98500 (to) ] TJ T* [ (in) 40.00560 (vestigate) -359.01400 (deeper) -358.01500 (or) -358.98800 (wider) -357.99200 (network) -359.01000 (ar) 36.98650 (c) 15.01220 (hitectur) 37.00360 (es\056) -636.00700 (Based) ] TJ T* [ (on) -263.98500 (the) -263.98700 (learnable) -264.98300 (activation) -265.01300 (and) -264.02000 (advanced) -263.98300 (initialization\054) -267.98200 (we) ] TJ T* [ (ac) 15.01770 (hie) 14.98590 (ve) ] TJ /R20 9.96260 Tf 32.90230 0 Td (4\05694\045) Tj /R18 9.96260 Tf 28.50780 0 Td [ (top\0555) -278.99400 (test) -277.89800 (err) 44.98020 (or) -279.00100 (on) -277.98300 (the) -279.00500 (Ima) 10.00320 (g) 10.00320 (eNet) -277.98100 (2012) -278.99300 (clas\055) ] TJ -61.41020 -11.95590 Td [ (si\223cation) -267.01000 (dataset\056) -361.98500 (This) -268 (is) -267 (a) -267.99000 (26\045) -267.00200 (r) 37.01960 (elative) -267 (impr) 44.99490 (o) 10.00320 (vement) -267.99200 (o) 10.00320 (ver) ] TJ T* [ (the) -235.01200 (ILSVRC) -234.98200 (2014) -235.90200 (winner) -234.98800 (\050Goo) 10.08660 (gLeNet\054) -238.00600 (6\05666\045) -235 (\13333\135\051\056) -304.98600 (T) 92 (o) -235.99500 (our) ] TJ T* [ (knowledg) 9.99098 (e) 9.99404 (\054) -388.99100 (our) -360.98300 (r) 37.01960 (esult) -361.01700 (is) -362.00500 (the) -360.99100 (\223r) 10.00940 (st) ] TJ /R16 6.97380 Tf 133.55700 3.61523 Td (1) Tj /R18 9.96260 Tf 7.58398 -3.61484 Td [ (to) -360.98600 (surpass) -360.99900 (the) -362.01100 (r) 37.01830 (eported) ] TJ -141.14100 -11.95510 Td [ (human\055le) 15.11820 (vel) -249.99900 (performance) -250.00700 (\0505\0561\045\054) -249.98400 (\13326\135\051) -250.01700 (on) -249.98800 (this) -250.01200 (dataset\056) ] TJ /R14 11.95520 Tf 25.00430 TL T* [ (1\056) -249.99000 (Intr) 18.01460 (oduction) ] TJ /R16 9.96260 Tf 11.95510 -18.92890 Td [ (Con) 40.01660 (v) 20.00160 (olutional) -459.99700 (neural) -459.99700 (netw) 10.00940 (orks) -459.98900 (\050CNNs\051) -459.98200 (\13319\054) -461.01100 (18\135) -459.99400 (ha) 19.99670 (v) 14.98280 (e) ] TJ -11.95510 -11.95510 Td [ (demonstrated) -270.89800 (recognition) -271.01800 (accurac) 15.11510 (y) -269.98900 (better) -271.00600 (than) -271.00100 (or) -271.01100 (compara\055) ] TJ 11.95510 TL T* [ (ble) -372.00800 (to) -372.00400 (humans) -371.98700 (in) -372.00400 (se) 25.01670 (v) 14.98280 (eral) -371.98200 (visual) -371.01000 (recognition) -371.98200 (tasks\054) -403.00900 (includ\055) ] TJ T* [ (ing) -324.00700 (recognizing) -322.90300 (traf) 25.01180 (\223c) -324.01500 (signs) -323.01500 (\1333\135\054) -341.98100 (f) 9.99466 (aces) -323.99200 (\13334\054) -322.91700 (32\135\054) -342.01200 (and) -324.01900 (hand\055) ] TJ T* [ (written) -278.99000 (digits) -279.99400 (\1333\054) -279.00900 (36\135\056) -398.00200 (In) -279.01100 (this) -280.00600 (w) 10 (ork\054) -285.98200 (we) -279.99500 (present) -279 (a) -279.01700 (result) -280.01500 (that) ] TJ T* [ (surpasses) -242.99600 (the) -243.99100 (human\055le) 24.99830 (v) 14.98280 (el) -243.99100 (performance) -243.01800 (reported) -243.99100 (by) -243.00800 (\13326\135) -243.99600 (on) ] TJ T* [ (a) -283.99600 (more) -283.00200 (generic) -283.99100 (and) -283.98600 (challenging) -283.01800 (recognition) -283.99700 (task) -282.99700 (\055) -283.99400 (the) -283.98400 (clas\055) ] TJ T* [ (si\223cation) -249.99300 (task) -249.98400 (in) -249.98500 (the) -249.99000 (1000\055class) -250.01000 (ImageNet) -249.98800 (dataset) -249.98500 (\13326\135\056) ] TJ 11.95510 -11.95510 Td [ (In) -245.99600 (the) -247.01000 (last) -245.98200 (fe) 25.00810 (w) -245.99600 (years\054) -247.01600 (we) -247.01900 (ha) 19.99790 (v) 14.98280 (e) -246.00300 (witnessed) -246.00600 (tremendous) -246.98300 (im\055) ] TJ -11.95510 -11.95510 Td [ (pro) 14.98460 (v) 14.98280 (ements) -306.98600 (in) -306.99500 (recognition) -307.99300 (performance\054) -321.00300 (mainly) -307.01500 (due) -307.00300 (to) -308.01500 (ad\055) ] TJ 11.95590 TL T* [ (v) 24.98110 (ances) -269.01600 (in) -269.00200 (tw) 10.00810 (o) -269.98900 (technical) -269 (directions\072) -347.98200 (b) 20.00160 (uilding) -269.01100 (more) -269.98400 (po) 24.98600 (werful) ] TJ -0.00100 Tc T* [ (models\054) -411.10800 (and) -379.09400 (designing) -378.09000 (ef) 24.88680 (fecti) 24.91010 (v) 14.88230 (e) -379.10000 (strate) 14.89700 (gies) -379.10800 (against) -379.08600 (o) 14.88230 (v) 14.88230 (er\223t\055) ] TJ 0 Tc 11.95510 TL T* [ (ting\056) -347.99300 (On) -261.98800 (one) -263.00900 (hand\054) -265.98100 (neural) -261.99100 (netw) 10.00940 (orks) -263.00300 (are) -262.98800 (becoming) -262.98300 (more) -263.00500 (ca\055) ] TJ T* [ (pable) -219.01400 (of) -220 (\223tting) -219.01600 (training) -220.01200 (data\054) -225.01500 (because) -220.00400 (of) -218.98100 (increased) -220 (comple) 14.99750 (x\055) ] TJ T* [ (ity) -288.01100 (\050) ] TJ -0.14944 Tc /R18 9.96260 Tf 16.70900 0 Td [ (e\056) -14.98460 (g) ] TJ 0 Tc /R16 9.96260 Tf 11.74610 0 Td [ (\056\054) -297.98500 (increased) -287.98900 (depth) -288.01200 (\13329\054) -287.98100 (33\135\054) -298.01900 (enlar) 17.99700 (ged) -287.98600 (width) -288.00300 (\13337\054) -287.98100 (28\135\054) ] TJ -28.45510 -11.95550 Td [ (and) -392.98800 (the) -392.00500 (use) -392.98700 (of) -392.01200 (smaller) -393.01200 (strides) -391.98800 (\13337\054) -392.98300 (28\054) -392 (2\054) -392.99100 (29\135\051\054) -428.00200 (ne) 25.01540 (w) -392.99100 (non\055) ] TJ 11.95590 TL T* [ (linear) -449.99600 (acti) 24.98170 (v) 24.98110 (ations) -450.01700 (\13324\054) -449.99400 (23\054) -449.01100 (38\054) -449.99100 (22\054) -449.99100 (31\054) -449.99100 (10\135\054) -499.98500 (and) -449.99800 (sophisti\055) ] TJ 11.95510 TL T* [ (cated) -505.99500 (layer) -505.91600 (designs) -506.00300 (\13333\054) -505.98400 (12\135\056) -1078.01000 (On) -505.98700 (the) -507.00600 (other) -505.98400 (hand\054) -570.01000 (bet\055) ] TJ T* [ (ter) -473.02000 (generalization) -473.89800 (is) -473.00600 (achie) 25.01420 (v) 14.98280 (ed) -473.99400 (by) -473.01400 (ef) 25.01180 (fecti) 25.01790 (v) 14.98280 (e) -474.00400 (re) 15.00980 (gularization) ] TJ ET Q /R22 gs 3.98000 w /R13 CS 1 SCN 501.12100 906.78100 m 1446.11000 906.78100 l S q 10 0 0 10 0 0 cm BT /R16 5.97760 Tf 1 0 0 1 60.97110 83.81170 Tm (1) Tj /R16 7.97010 Tf 3.48711 -2.81328 Td [ (reported) -250.08900 (in) -250.01200 (Feb) 40.00240 (\056) -310.00300 (2015\056) ] TJ /R16 9.96260 Tf 244.40400 481.47000 Td [ (techniques) -351.01300 (\13313\054) -350.01000 (30\054) -350.98800 (10\054) -350.99300 (36\135\054) -374.98600 (aggressi) 24.99580 (v) 14.98280 (e) -351.00500 (data) -350.99500 (augmentation) ] TJ 11.95510 TL T* [ (\13318\054) -249.98800 (14\054) -249.98500 (29\054) -249.98500 (33\135\054) -249.98800 (and) -249.99300 (lar) 17.99700 (ge\055scale) -249.99700 (data) -249.99300 (\1334\054) -249.99300 (26\135\056) ] TJ 11.95510 -12.77110 Td [ (Among) -355.91100 (these) -356.98900 (adv) 24.98110 (ances\054) -382.01500 (the) -356.98900 (recti\223er) -355.89200 (neuron) -356.00400 (\13324\054) -356.00900 (9\054) -357.10700 (23\054) ] TJ -11.95510 -11.95510 Td (38\135\054) Tj -0.14944 Tc /R18 9.96260 Tf 19.71210 0 Td [ (e\056) -14.98520 (g) ] TJ 0 Tc /R16 9.96260 Tf 11.74610 0 Td [ (\056\054) -396.00700 (Recti\223ed) -365.98300 (Linear) -366.99300 (Unit) -365.99300 (\050ReLU\051\054) -366.98800 (is) -366.00300 (one) -366.99300 (of) -366.01700 (se) 25.01540 (v) 14.98280 (eral) ] TJ -31.45820 -11.95550 Td [ (k) 10.00320 (e) 15.01220 (ys) -316.01300 (to) -316.01300 (the) -316.01800 (recent) -317.01100 (success) -316.01600 (of) -315.98600 (deep) -315.98600 (netw) 10.00810 (orks) -316.01100 (\13318\135\056) -509.00800 (It) -315.98200 (e) 15.01220 (xpe\055) ] TJ T* [ (dites) -314.00400 (con) 40.11590 (v) 14.98280 (er) 17.99450 (gence) -315.00900 (of) -313.98400 (the) -314.99900 (training) -313.99700 (procedure) -315.00600 (\13318\135) -313.98200 (and) -315.00100 (leads) ] TJ T* [ (to) -193.99500 (better) -194 (solutions) -193.99200 (\13324\054) -193.09500 (9\054) -194 (23\054) -193.01200 (38\135) -194.00200 (than) -193.01200 (con) 39.99820 (v) 14.98280 (entional) -194.01700 (sigmoid\055) ] TJ T* [ (lik) 10.01790 (e) -450.98800 (units\056) -914.01200 (Despite) -451.98300 (the) -451.99600 (pre) 25.01050 (v) 24.98110 (alence) -450.98800 (of) -450.98400 (recti\223er) -451.99300 (netw) 10.00810 (orks\054) ] TJ T* [ (recent) -416.01400 (impro) 15.00480 (v) 14.98280 (ements) -417.00600 (of) -416.00700 (models) -416.01900 (\13337\054) -416 (28\054) -417.01400 (12\054) -415.99400 (29\054) -415.99400 (33\135) -416.98400 (and) ] TJ T* [ (theoretical) -312.91100 (guidelines) -313.00700 (for) -312.88900 (training) -313.01600 (them) -312.98700 (\1338\054) -313.00200 (27\135) -313.00700 (ha) 19.99670 (v) 14.98280 (e) -313.01200 (rarely) ] TJ T* [ (focused) -249.90900 (on) -249.98800 (the) -249.99000 (properties) -250.01700 (of) -249.99700 (the) -249.98800 (recti\223ers\056) ] TJ 11.95510 -12.77110 Td [ (Unlik) 9.98608 (e) -222.00700 (traditional) -222.98200 (sigmoid\055lik) 9.98363 (e) -222.98700 (units\054) ] TJ /R18 9.96260 Tf 149.02600 0 Td [ (ReLU) -221.90900 (is) -223.00700 (not) -221.98300 (a) -221.99700 (sym\055) ] TJ -160.98100 -11.95510 Td [ (metric) -340.98700 (function) ] TJ /R16 9.96260 Tf 61.51210 0 Td [ (\056) -584.01800 (As) -342.01600 (a) -341.00700 (consequence\054) -364.00800 (the) -342.01200 (mean) -340.99200 (response) -340.98700 (of) ] TJ -61.51210 -11.95510 Td [ (ReLU) -255.98200 (is) -255.00400 (al) 10 (w) 10 (ays) -255.98900 (no) -255.98700 (smaller) -256.01600 (than) -255.00200 (zero\073) -259.01600 (besides\054) -256.99100 (e) 25.01050 (v) 14.98280 (en) -255.99600 (assum\055) ] TJ T* [ (ing) -421.91200 (the) -423.02000 (inputs\057weights) -421.89800 (are) -422.99800 (subject) -421.91000 (to) -421.99800 (symmetric) -422.98800 (distrib) 19.99180 (u\055) ] TJ T* [ (tions\054) -264.00300 (the) -261.98800 (distrib) 19.99180 (utions) -262 (of) -261.01300 (responses) -262.01500 (can) -262 (still) -262 (be) -262 (asymmetric) ] TJ T* [ (because) -202.91000 (of) -202.00300 (the) -203.01800 (beha) 20 (vior) -203.01000 (of) -202.00600 (ReLU\056) -203.00500 (These) -202 (properties) -203.00500 (of) -202.98600 (ReLU) ] TJ T* [ (in\224uence) -211 (the) -211.91800 (theoretical) -211.00600 (analysis) -210.91600 (of) -212.00400 (con) 39.99820 (v) 14.98280 (er) 17.99210 (gence) -210.98900 (and) -211.99900 (empir) 20.01140 (\055) ] TJ T* [ (ical) -249.99300 (performance\054) -249.91700 (as) -249.99500 (we) -250 (will) -250.01200 (demonstrate\056) ] TJ -0.00100 Tc 11.95510 -12.77070 Td [ (In) -344.10200 (this) -344.08200 (paper) 39.88300 (\054) -368.11500 (w) -0.09802 (e) -344.08700 (i) -0.09312 (n) 39.89780 (v) 14.88230 (estigate) -344.09500 (neural) -344.10400 (netw) 9.88560 (orks) -345.11400 (from) -344.09500 (tw) 9.88560 (o) ] TJ 0 Tc -11.95510 -11.95550 Td [ (aspects) -275 (particularly) -274.91000 (dri) 24.98600 (v) 14.98280 (en) -275.00800 (by) -275.00800 (the) -275.00300 (recti\223er) -275.00300 (properties\056) -385.99000 (First\054) ] TJ T* [ (we) -543.99000 (propose) -544.00700 (a) -545.01200 (ne) 25.01050 (w) -543.98500 (e) 15.01220 (xtension) -543.99300 (of) -543.98800 (ReLU\054) -544.00700 (which) -544.99200 (we) -543.98800 (call) ] TJ /R18 9.96260 Tf T* [ (P) 79.99160 (ar) 14.98280 (ametric) -284.01100 (Recti\223ed) -284.01600 (Linear) -283.99700 (Unit) ] TJ /R16 9.96260 Tf 135.07100 0 Td [ (\050PReLU\051\056) -284.00700 (This) -284.00700 (acti) 24.98110 (v) 24.98110 (ation) ] TJ -135.07100 -11.95590 Td [ (function) -350.89500 (adapti) 24.99090 (v) 14.98280 (ely) -350.01300 (learns) -350.99500 (the) -350.01000 (parameters) -350.99500 (of) -350.02000 (the) -350.99000 (recti\223ers\054) ] TJ T* [ (and) -438 (impro) 15.00480 (v) 14.98280 (es) -438 (accurac) 14.99750 (y) -436.98100 (at) -438 (ne) 15.01220 (gligible) -438.00500 (e) 15.01220 (xtra) -438 (computational) ] TJ T* [ (cost\056) -655.99100 (Second\054) -393.99100 (we) -365 (study) -365.00800 (the) -364.98800 (dif) 24.98600 (\223culty) -365.00300 (of) -366.01700 (training) -365.00800 (recti\223ed) ] TJ T* [ (models) -261.88500 (that) -261.98100 (are) -262.00800 (v) 14.98280 (ery) -261.99800 (deep\056) -344.99600 (By) -262.01500 (e) 15.01220 (xplicitly) -261.99100 (modeling) -261.99100 (the) -261.98600 (non\055) ] TJ T* [ (linearity) -318.90000 (of) -319.00300 (recti\223ers) -318.01100 (\050ReLU\057PReLU\051\054) -318.99600 (we) -319.00600 (deri) 24.98110 (v) 14.98280 (e) -319.01000 (a) -319.01000 (theoret\055) ] TJ T* [ (ically) -379.01800 (sound) -378.89000 (initialization) -379.99500 (method\054) -410.91700 (which) -378.90200 (helps) -379.01500 (with) -380.10800 (con\055) ] TJ T* [ (v) 14.98280 (er) 17.99450 (gence) -281.01400 (of) -279.99000 (v) 14.98280 (ery) -281.01400 (deep) -279.99000 (models) -280.98300 (\050) ] TJ -0.14944 Tc /R18 9.96260 Tf 126.36400 0 Td [ (e\056) -14.98280 (g) ] TJ 0 Tc /R16 9.96260 Tf 11.74610 0 Td [ (\056\054) -287.98600 (with) -280.98700 (30) -279.98300 (weight) -280.98700 (layers\051) ] TJ -138.11000 -11.95510 Td [ (trained) -262.88500 (directly) -262.99500 (from) -261.90700 (scratch\056) -348.99600 (This) -262.01000 (gi) 24.98600 (v) 14.98280 (es) -263.01000 (us) -263.01000 (more) -263.00500 (\224e) 14.98770 (xibility) ] TJ T* [ (to) -249.98500 (e) 15.01220 (xplore) -249.98300 (more) -249.98800 (po) 24.98600 (werful) -249.99300 (netw) 10.00810 (ork) -249.99300 (architectures\056) ] TJ 11.95510 -12.77110 Td [ (On) -314.99900 (the) -314.99900 (1000\055class) -315.99900 (ImageNet) -314.99600 (2012) -314.98700 (dataset\054) -330.98900 (our) -316.02100 (netw) 10.00810 (ork) ] TJ -11.95510 -11.95510 Td [ (leads) -316.91800 (to) -316.99400 (a) -317.01100 (single\055model) -316.99600 (result) -316.98600 (of) -317.01100 (5\05671\045) -316.98600 (top\0555) -316.98600 (error) 39.98350 (\054) -333.97900 (which) ] TJ T* [ (surpasses) -392.89300 (all) -392.00300 (multi\055model) -392.89500 (results) -393.00800 (in) -392.98300 (ILSVRC) -392.00800 (2014\056) -738.98900 (Fur) 20.01140 (\055) ] TJ T* [ (ther) 40.11590 (\054) -287.00800 (our) -279.00500 (multi\055model) -278.99000 (result) -278.99300 (achie) 25.01540 (v) 14.98280 (es) ] TJ /R14 9.96260 Tf 150.15400 0 Td (4\05694\045) Tj /R16 9.96260 Tf 30.17970 0 Td [ (top\0555) -278.99300 (error) -279.88500 (on) ] TJ -180.33400 -11.95590 Td [ (the) -311 (test) -309.89200 (set\054) -326.01200 (which) -310.99200 (is) -310.99500 (a) -311.01200 (26\045) -310.01700 (relati) 24.98600 (v) 14.98280 (e) -311.01200 (impro) 15.00240 (v) 14.98280 (ement) -310.99200 (o) 14.98280 (v) 14.98280 (er) -311.01200 (the) ] TJ 11.95630 TL T* [ (ILSVRC) -378.01000 (2014) -377.01600 (winner) -378.00300 (\050GoogLeNet\054) -409.90700 (6\05666\045) -377.91700 (\13333\135\051\056) -693.01700 (T) 79.99160 (o) -378.01000 (the) ] TJ T* [ (best) -225.90800 (of) -227.01800 (our) -227.01400 (kno) 24.99090 (wledge\054) -231.00600 (our) -225.99700 (result) -227.00100 (surpasses) -226.01600 (for) -226.98200 (the) -225.99200 (\223rst) -226.99600 (time) ] TJ 11.95550 TL T* [ (the) -334.99600 (reported) -334.01500 (human\055le) 25.11590 (v) 14.98280 (el) -334.99800 (performance) -335.00300 (\0505\0561\045) -334.99300 (in) -334.01300 (\13326\135\051) -335.00300 (of) -335.00300 (a) ] TJ T* [ (dedicated) -249.90700 (indi) 25 (vidual) -250.01200 (labeler) -249.99300 (on) -249.98800 (this) -250.01200 (recognition) -250.00200 (challenge\056) ] TJ -13.74140 -29.88750 Td (1) Tj ET Q /R23 gs Q Q q q 1 1 1 rg /a0 gs 48.40600 786.42200 515.18800 -52.69900 re f q /s5 gs /x6 Do Q q /s7 gs /x8 Do Q q /s9 gs /x10 Do Q q /s11 gs /x12 Do Q Q Q Q q 1 0 0 1 0 0 cm BT /F1 12 Tf 14.40000 TL ET 1 1 1 rg n 270 47 72 14 re f* 0.50000 0.50000 0.50000 rg BT /F2 9 Tf 10.80000 TL ET BT 1 0 0 1 297 50 Tm (1026) Tj T* ET Q endstream endobj 14 0 obj << /Filter /FlateDecode /Resources << /ExtGState << /a0 << /CA 1 /ca 1 >> >> /XObject << /x18 15 0 R >> >> /Length 28 /Group << /Type /Group /S /Transparency /CS /DeviceRGB /I true >> /BBox [ 78 746 96 765 ] /Type /XObject /Subtype /Form >> stream x+O4PH/VЯ0Pp 0 endstream endobj 15 0 obj << /Filter /FlateDecode /Resources 16 0 R /Length 107 /Type /XObject /BBox [ 78 746 96 765 ] /Subtype /Form >> stream xe AC̬wʠ =p,?]%+H-
Jc "82w8VSnGW;"
endstream
endobj
16 0 obj
<<
/ExtGState <<
/a0 <<
/CA 1
/ca 1
>>
>>
>>
endobj
17 0 obj
<<
/Filter /FlateDecode
/Resources <<
/ExtGState <<
/a0 <<
/CA 1
/ca 1
>>
>>
/XObject <<
/x15 18 0 R
>>
>>
/Length 28
/Group <<
/Type /Group
/S /Transparency
/CS /DeviceRGB
/I true
>>
/BBox [ 67 752 84 775 ]
/Type /XObject
/Subtype /Form
>>
stream
x+O4PH/VЯ04Up
0
endstream
endobj
18 0 obj
<<
/Filter /FlateDecode
/Resources 19 0 R
/Length 228
/Type /XObject
/BBox [ 67 752 84 775 ]
/Subtype /Form
>>
stream
xeQKn!s ?FPav6R٪TS.
b];15YyR
{7QL.\:Rv/x9l+L7h%1!}i/AI(kz"U&,YO![R hg{3}4/GyYF:!w}Gn+'xJcO9i뽼_-:`
endstream
endobj
19 0 obj
<<
/ExtGState <<
/a0 <<
/CA 1
/ca 1
>>
>>
>>
endobj
20 0 obj
<<
/Filter /FlateDecode
/Resources <<
/ExtGState <<
/a0 <<
/CA 1
/ca 1
>>
>>
/XObject <<
/x24 21 0 R
>>
>>
/Length 28
/Group <<
/Type /Group
/S /Transparency
/CS /DeviceRGB
/I true
>>
/BBox [ 133 751 479 772 ]
/Type /XObject
/Subtype /Form
>>
stream
x+O4PH/VЯ02Qp
0
endstream
endobj
21 0 obj
<<
/Filter /FlateDecode
/Resources 22 0 R
/Length 53008
/Type /XObject
/BBox [ 133 751 479 772 ]
/Subtype /Form
>>
stream
xlK8,8?DKs9mavd
{f-8*2Y@H >חk}y}uf`v)_s}1z#*Gw_gX jow\o'1c|Z^G<
A̺X}ay?IT|y~L.[ {Ȟb\3-3]_'X\竵0{+_۾oY-wj+ B;)Aa=/
1~a(>}m_K' >*Q1:?OPnGsQ_`?mzN
?WBB!lU n'S 1Ipzχ1xF_09,}?e){:̹~Gu7g {0mD`pdS)>;ys`Pµx'O
+혝 oI~!p7̵&m=dC+>5ALV"I1"S*CKha#03L4|@p灟vMRQ==*/m"r>@uqo9ͻ;Vz 'ȳ;IBzWR~ihwa8PlT>]fZ8y52)r빃W{B7G/q)}M]elS(k=5^A] сڸc'w!Ll'M=Ȥ뎓z;#կN>\q6r>|8}}Lr$Mo5:>syls+Ǵ`C?/9{Y||jD u#i^\rJ8a|oY+Rxl<{[؏s迨nYaյ[?/^u;+ ,bhL}wrj:ɴdM8^q|m'aQ6;0k_ar|E8HɃpN('&r`V;Rs볩Nk\&W=DDG#5]~M{VyCSRH9ܜXˏt7߾i9ADگ
USڝ|{(hbIRZXg @6Hɹ0Dq;si\Tv<1?mUŏV7bF
p7Y9sƨ*.uz_LբřsB݃ɹ^'_Yae91f+n%A̶d
Ċc#q
CF=Mt'\&ξ)z;L-4:n}41j:mӶ_)\݇ |T@uAƓxAN6Ql;}Ɲ{~_y"g%]nCi']I~,xa?feoɌr0u?V=phldx.Y;~nZL603> \`ęܓ