Black Lives Matter. Support the Equal Justice Initiative.

Text file src/crypto/sha256/sha256block_amd64.s

Documentation: crypto/sha256

     1  // Copyright 2013 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  #include "textflag.h"
     6  
     7  // SHA256 block routine. See sha256block.go for Go equivalent.
     8  //
     9  // The algorithm is detailed in FIPS 180-4:
    10  //
    11  //  https://csrc.nist.gov/publications/fips/fips180-4/fips-180-4.pdf
    12  
    13  // The avx2-version is described in an Intel White-Paper:
    14  // "Fast SHA-256 Implementations on Intel Architecture Processors"
    15  // To find it, surf to http://www.intel.com/p/en_US/embedded
    16  // and search for that title.
    17  // AVX2 version by Intel, same algorithm as code in Linux kernel:
    18  // https://github.com/torvalds/linux/blob/master/arch/x86/crypto/sha256-avx2-asm.S
    19  // by
    20  //     James Guilford <james.guilford@intel.com>
    21  //     Kirk Yap <kirk.s.yap@intel.com>
    22  //     Tim Chen <tim.c.chen@linux.intel.com>
    23  
    24  // Wt = Mt; for 0 <= t <= 15
    25  // Wt = SIGMA1(Wt-2) + SIGMA0(Wt-15) + Wt-16; for 16 <= t <= 63
    26  //
    27  // a = H0
    28  // b = H1
    29  // c = H2
    30  // d = H3
    31  // e = H4
    32  // f = H5
    33  // g = H6
    34  // h = H7
    35  //
    36  // for t = 0 to 63 {
    37  //    T1 = h + BIGSIGMA1(e) + Ch(e,f,g) + Kt + Wt
    38  //    T2 = BIGSIGMA0(a) + Maj(a,b,c)
    39  //    h = g
    40  //    g = f
    41  //    f = e
    42  //    e = d + T1
    43  //    d = c
    44  //    c = b
    45  //    b = a
    46  //    a = T1 + T2
    47  // }
    48  //
    49  // H0 = a + H0
    50  // H1 = b + H1
    51  // H2 = c + H2
    52  // H3 = d + H3
    53  // H4 = e + H4
    54  // H5 = f + H5
    55  // H6 = g + H6
    56  // H7 = h + H7
    57  
    58  // Wt = Mt; for 0 <= t <= 15
    59  #define MSGSCHEDULE0(index) \
    60  	MOVL	(index*4)(SI), AX; \
    61  	BSWAPL	AX; \
    62  	MOVL	AX, (index*4)(BP)
    63  
    64  // Wt = SIGMA1(Wt-2) + Wt-7 + SIGMA0(Wt-15) + Wt-16; for 16 <= t <= 63
    65  //   SIGMA0(x) = ROTR(7,x) XOR ROTR(18,x) XOR SHR(3,x)
    66  //   SIGMA1(x) = ROTR(17,x) XOR ROTR(19,x) XOR SHR(10,x)
    67  #define MSGSCHEDULE1(index) \
    68  	MOVL	((index-2)*4)(BP), AX; \
    69  	MOVL	AX, CX; \
    70  	RORL	$17, AX; \
    71  	MOVL	CX, DX; \
    72  	RORL	$19, CX; \
    73  	SHRL	$10, DX; \
    74  	MOVL	((index-15)*4)(BP), BX; \
    75  	XORL	CX, AX; \
    76  	MOVL	BX, CX; \
    77  	XORL	DX, AX; \
    78  	RORL	$7, BX; \
    79  	MOVL	CX, DX; \
    80  	SHRL	$3, DX; \
    81  	RORL	$18, CX; \
    82  	ADDL	((index-7)*4)(BP), AX; \
    83  	XORL	CX, BX; \
    84  	XORL	DX, BX; \
    85  	ADDL	((index-16)*4)(BP), BX; \
    86  	ADDL	BX, AX; \
    87  	MOVL	AX, ((index)*4)(BP)
    88  
    89  // Calculate T1 in AX - uses AX, CX and DX registers.
    90  // h is also used as an accumulator. Wt is passed in AX.
    91  //   T1 = h + BIGSIGMA1(e) + Ch(e, f, g) + Kt + Wt
    92  //     BIGSIGMA1(x) = ROTR(6,x) XOR ROTR(11,x) XOR ROTR(25,x)
    93  //     Ch(x, y, z) = (x AND y) XOR (NOT x AND z)
    94  #define SHA256T1(const, e, f, g, h) \
    95  	ADDL	AX, h; \
    96  	MOVL	e, AX; \
    97  	ADDL	$const, h; \
    98  	MOVL	e, CX; \
    99  	RORL	$6, AX; \
   100  	MOVL	e, DX; \
   101  	RORL	$11, CX; \
   102  	XORL	CX, AX; \
   103  	MOVL	e, CX; \
   104  	RORL	$25, DX; \
   105  	ANDL	f, CX; \
   106  	XORL	AX, DX; \
   107  	MOVL	e, AX; \
   108  	NOTL	AX; \
   109  	ADDL	DX, h; \
   110  	ANDL	g, AX; \
   111  	XORL	CX, AX; \
   112  	ADDL	h, AX
   113  
   114  // Calculate T2 in BX - uses BX, CX, DX and DI registers.
   115  //   T2 = BIGSIGMA0(a) + Maj(a, b, c)
   116  //     BIGSIGMA0(x) = ROTR(2,x) XOR ROTR(13,x) XOR ROTR(22,x)
   117  //     Maj(x, y, z) = (x AND y) XOR (x AND z) XOR (y AND z)
   118  #define SHA256T2(a, b, c) \
   119  	MOVL	a, DI; \
   120  	MOVL	c, BX; \
   121  	RORL	$2, DI; \
   122  	MOVL	a, DX; \
   123  	ANDL	b, BX; \
   124  	RORL	$13, DX; \
   125  	MOVL	a, CX; \
   126  	ANDL	c, CX; \
   127  	XORL	DX, DI; \
   128  	XORL	CX, BX; \
   129  	MOVL	a, DX; \
   130  	MOVL	b, CX; \
   131  	RORL	$22, DX; \
   132  	ANDL	a, CX; \
   133  	XORL	CX, BX; \
   134  	XORL	DX, DI; \
   135  	ADDL	DI, BX
   136  
   137  // Calculate T1 and T2, then e = d + T1 and a = T1 + T2.
   138  // The values for e and a are stored in d and h, ready for rotation.
   139  #define SHA256ROUND(index, const, a, b, c, d, e, f, g, h) \
   140  	SHA256T1(const, e, f, g, h); \
   141  	SHA256T2(a, b, c); \
   142  	MOVL	BX, h; \
   143  	ADDL	AX, d; \
   144  	ADDL	AX, h
   145  
   146  #define SHA256ROUND0(index, const, a, b, c, d, e, f, g, h) \
   147  	MSGSCHEDULE0(index); \
   148  	SHA256ROUND(index, const, a, b, c, d, e, f, g, h)
   149  
   150  #define SHA256ROUND1(index, const, a, b, c, d, e, f, g, h) \
   151  	MSGSCHEDULE1(index); \
   152  	SHA256ROUND(index, const, a, b, c, d, e, f, g, h)
   153  
   154  
   155  // Definitions for AVX2 version
   156  
   157  // addm (mem), reg
   158  // Add reg to mem using reg-mem add and store
   159  #define addm(P1, P2) \
   160  	ADDL P2, P1; \
   161  	MOVL P1, P2
   162  
   163  #define XDWORD0 Y4
   164  #define XDWORD1 Y5
   165  #define XDWORD2 Y6
   166  #define XDWORD3 Y7
   167  
   168  #define XWORD0 X4
   169  #define XWORD1 X5
   170  #define XWORD2 X6
   171  #define XWORD3 X7
   172  
   173  #define XTMP0 Y0
   174  #define XTMP1 Y1
   175  #define XTMP2 Y2
   176  #define XTMP3 Y3
   177  #define XTMP4 Y8
   178  #define XTMP5 Y11
   179  
   180  #define XFER  Y9
   181  
   182  #define BYTE_FLIP_MASK 	Y13 // mask to convert LE -> BE
   183  #define X_BYTE_FLIP_MASK X13
   184  
   185  #define NUM_BYTES DX
   186  #define INP	DI
   187  
   188  #define CTX SI // Beginning of digest in memory (a, b, c, ... , h)
   189  
   190  #define a AX
   191  #define b BX
   192  #define c CX
   193  #define d R8
   194  #define e DX
   195  #define f R9
   196  #define g R10
   197  #define h R11
   198  
   199  #define old_h R11
   200  
   201  #define TBL BP
   202  
   203  #define SRND SI // SRND is same register as CTX
   204  
   205  #define T1 R12
   206  
   207  #define y0 R13
   208  #define y1 R14
   209  #define y2 R15
   210  #define y3 DI
   211  
   212  // Offsets
   213  #define XFER_SIZE 2*64*4
   214  #define INP_END_SIZE 8
   215  #define INP_SIZE 8
   216  
   217  #define _XFER 0
   218  #define _INP_END _XFER + XFER_SIZE
   219  #define _INP _INP_END + INP_END_SIZE
   220  #define STACK_SIZE _INP + INP_SIZE
   221  
   222  #define ROUND_AND_SCHED_N_0(disp, a, b, c, d, e, f, g, h, XDWORD0, XDWORD1, XDWORD2, XDWORD3) \
   223  	;                                     \ // #############################  RND N + 0 ############################//
   224  	MOVL     a, y3;                       \ // y3 = a					// MAJA
   225  	RORXL    $25, e, y0;                  \ // y0 = e >> 25				// S1A
   226  	RORXL    $11, e, y1;                  \ // y1 = e >> 11				// S1B
   227  	;                                     \
   228  	ADDL     (disp + 0*4)(SP)(SRND*1), h; \ // h = k + w + h        // disp = k + w
   229  	ORL      c, y3;                       \ // y3 = a|c				// MAJA
   230  	VPALIGNR $4, XDWORD2, XDWORD3, XTMP0; \ // XTMP0 = W[-7]
   231  	MOVL     f, y2;                       \ // y2 = f				// CH
   232  	RORXL    $13, a, T1;                  \ // T1 = a >> 13			// S0B
   233  	;                                     \
   234  	XORL     y1, y0;                      \ // y0 = (e>>25) ^ (e>>11)					// S1
   235  	XORL     g, y2;                       \ // y2 = f^g                              	// CH
   236  	VPADDD   XDWORD0, XTMP0, XTMP0;       \ // XTMP0 = W[-7] + W[-16]	// y1 = (e >> 6)	// S1
   237  	RORXL    $6, e, y1;                   \ // y1 = (e >> 6)						// S1
   238  	;                                     \
   239  	ANDL     e, y2;                       \ // y2 = (f^g)&e                         // CH
   240  	XORL     y1, y0;                      \ // y0 = (e>>25) ^ (e>>11) ^ (e>>6)		// S1
   241  	RORXL    $22, a, y1;                  \ // y1 = a >> 22							// S0A
   242  	ADDL     h, d;                        \ // d = k + w + h + d                     	// --
   243  	;                                     \
   244  	ANDL     b, y3;                       \ // y3 = (a|c)&b							// MAJA
   245  	VPALIGNR $4, XDWORD0, XDWORD1, XTMP1; \ // XTMP1 = W[-15]
   246  	XORL     T1, y1;                      \ // y1 = (a>>22) ^ (a>>13)				// S0
   247  	RORXL    $2, a, T1;                   \ // T1 = (a >> 2)						// S0
   248  	;                                     \
   249  	XORL     g, y2;                       \ // y2 = CH = ((f^g)&e)^g				// CH
   250  	VPSRLD   $7, XTMP1, XTMP2;            \
   251  	XORL     T1, y1;                      \ // y1 = (a>>22) ^ (a>>13) ^ (a>>2)		// S0
   252  	MOVL     a, T1;                       \ // T1 = a								// MAJB
   253  	ANDL     c, T1;                       \ // T1 = a&c								// MAJB
   254  	;                                     \
   255  	ADDL     y0, y2;                      \ // y2 = S1 + CH							// --
   256  	VPSLLD   $(32-7), XTMP1, XTMP3;       \
   257  	ORL      T1, y3;                      \ // y3 = MAJ = (a|c)&b)|(a&c)			// MAJ
   258  	ADDL     y1, h;                       \ // h = k + w + h + S0					// --
   259  	;                                     \
   260  	ADDL     y2, d;                       \ // d = k + w + h + d + S1 + CH = d + t1  // --
   261  	VPOR     XTMP2, XTMP3, XTMP3;         \ // XTMP3 = W[-15] ror 7
   262  	;                                     \
   263  	VPSRLD   $18, XTMP1, XTMP2;           \
   264  	ADDL     y2, h;                       \ // h = k + w + h + S0 + S1 + CH = t1 + S0// --
   265  	ADDL     y3, h                        // h = t1 + S0 + MAJ                     // --
   266  
   267  #define ROUND_AND_SCHED_N_1(disp, a, b, c, d, e, f, g, h, XDWORD0, XDWORD1, XDWORD2, XDWORD3) \
   268  	;                                    \ // ################################### RND N + 1 ############################
   269  	;                                    \
   270  	MOVL    a, y3;                       \ // y3 = a                       // MAJA
   271  	RORXL   $25, e, y0;                  \ // y0 = e >> 25					// S1A
   272  	RORXL   $11, e, y1;                  \ // y1 = e >> 11					// S1B
   273  	ADDL    (disp + 1*4)(SP)(SRND*1), h; \ // h = k + w + h         		// --
   274  	ORL     c, y3;                       \ // y3 = a|c						// MAJA
   275  	;                                    \
   276  	VPSRLD  $3, XTMP1, XTMP4;            \ // XTMP4 = W[-15] >> 3
   277  	MOVL    f, y2;                       \ // y2 = f						// CH
   278  	RORXL   $13, a, T1;                  \ // T1 = a >> 13					// S0B
   279  	XORL    y1, y0;                      \ // y0 = (e>>25) ^ (e>>11)		// S1
   280  	XORL    g, y2;                       \ // y2 = f^g						// CH
   281  	;                                    \
   282  	RORXL   $6, e, y1;                   \ // y1 = (e >> 6)				// S1
   283  	XORL    y1, y0;                      \ // y0 = (e>>25) ^ (e>>11) ^ (e>>6)	// S1
   284  	RORXL   $22, a, y1;                  \ // y1 = a >> 22						// S0A
   285  	ANDL    e, y2;                       \ // y2 = (f^g)&e						// CH
   286  	ADDL    h, d;                        \ // d = k + w + h + d				// --
   287  	;                                    \
   288  	VPSLLD  $(32-18), XTMP1, XTMP1;      \
   289  	ANDL    b, y3;                       \ // y3 = (a|c)&b					// MAJA
   290  	XORL    T1, y1;                      \ // y1 = (a>>22) ^ (a>>13)		// S0
   291  	;                                    \
   292  	VPXOR   XTMP1, XTMP3, XTMP3;         \
   293  	RORXL   $2, a, T1;                   \ // T1 = (a >> 2)				// S0
   294  	XORL    g, y2;                       \ // y2 = CH = ((f^g)&e)^g		// CH
   295  	;                                    \
   296  	VPXOR   XTMP2, XTMP3, XTMP3;         \ // XTMP3 = W[-15] ror 7 ^ W[-15] ror 18
   297  	XORL    T1, y1;                      \ // y1 = (a>>22) ^ (a>>13) ^ (a>>2)		// S0
   298  	MOVL    a, T1;                       \ // T1 = a						// MAJB
   299  	ANDL    c, T1;                       \ // T1 = a&c						// MAJB
   300  	ADDL    y0, y2;                      \ // y2 = S1 + CH					// --
   301  	;                                    \
   302  	VPXOR   XTMP4, XTMP3, XTMP1;         \ // XTMP1 = s0
   303  	VPSHUFD $0xFA, XDWORD3, XTMP2;       \ // XTMP2 = W[-2] {BBAA}
   304  	ORL     T1, y3;                      \ // y3 = MAJ = (a|c)&b)|(a&c)             // MAJ
   305  	ADDL    y1, h;                       \ // h = k + w + h + S0                    // --
   306  	;                                    \
   307  	VPADDD  XTMP1, XTMP0, XTMP0;         \ // XTMP0 = W[-16] + W[-7] + s0
   308  	ADDL    y2, d;                       \ // d = k + w + h + d + S1 + CH = d + t1  // --
   309  	ADDL    y2, h;                       \ // h = k + w + h + S0 + S1 + CH = t1 + S0// --
   310  	ADDL    y3, h;                       \ // h = t1 + S0 + MAJ                     // --
   311  	;                                    \
   312  	VPSRLD  $10, XTMP2, XTMP4            // XTMP4 = W[-2] >> 10 {BBAA}
   313  
   314  #define ROUND_AND_SCHED_N_2(disp, a, b, c, d, e, f, g, h, XDWORD0, XDWORD1, XDWORD2, XDWORD3) \
   315  	;                                    \ // ################################### RND N + 2 ############################
   316  	;                                    \
   317  	MOVL    a, y3;                       \ // y3 = a							// MAJA
   318  	RORXL   $25, e, y0;                  \ // y0 = e >> 25						// S1A
   319  	ADDL    (disp + 2*4)(SP)(SRND*1), h; \ // h = k + w + h        			// --
   320  	;                                    \
   321  	VPSRLQ  $19, XTMP2, XTMP3;           \ // XTMP3 = W[-2] ror 19 {xBxA}
   322  	RORXL   $11, e, y1;                  \ // y1 = e >> 11						// S1B
   323  	ORL     c, y3;                       \ // y3 = a|c                         // MAJA
   324  	MOVL    f, y2;                       \ // y2 = f                           // CH
   325  	XORL    g, y2;                       \ // y2 = f^g                         // CH
   326  	;                                    \
   327  	RORXL   $13, a, T1;                  \ // T1 = a >> 13						// S0B
   328  	XORL    y1, y0;                      \ // y0 = (e>>25) ^ (e>>11)			// S1
   329  	VPSRLQ  $17, XTMP2, XTMP2;           \ // XTMP2 = W[-2] ror 17 {xBxA}
   330  	ANDL    e, y2;                       \ // y2 = (f^g)&e						// CH
   331  	;                                    \
   332  	RORXL   $6, e, y1;                   \ // y1 = (e >> 6)					// S1
   333  	VPXOR   XTMP3, XTMP2, XTMP2;         \
   334  	ADDL    h, d;                        \ // d = k + w + h + d				// --
   335  	ANDL    b, y3;                       \ // y3 = (a|c)&b						// MAJA
   336  	;                                    \
   337  	XORL    y1, y0;                      \ // y0 = (e>>25) ^ (e>>11) ^ (e>>6)	// S1
   338  	RORXL   $22, a, y1;                  \ // y1 = a >> 22						// S0A
   339  	VPXOR   XTMP2, XTMP4, XTMP4;         \ // XTMP4 = s1 {xBxA}
   340  	XORL    g, y2;                       \ // y2 = CH = ((f^g)&e)^g			// CH
   341  	;                                    \
   342  	VPSHUFB shuff_00BA<>(SB), XTMP4, XTMP4;\ // XTMP4 = s1 {00BA}
   343  	;                                    \
   344  	XORL    T1, y1;                      \ // y1 = (a>>22) ^ (a>>13)		// S0
   345  	RORXL   $2, a, T1;                   \ // T1 = (a >> 2)				// S0
   346  	VPADDD  XTMP4, XTMP0, XTMP0;         \ // XTMP0 = {..., ..., W[1], W[0]}
   347  	;                                    \
   348  	XORL    T1, y1;                      \ // y1 = (a>>22) ^ (a>>13) ^ (a>>2)	// S0
   349  	MOVL    a, T1;                       \ // T1 = a                                // MAJB
   350  	ANDL    c, T1;                       \ // T1 = a&c                              // MAJB
   351  	ADDL    y0, y2;                      \ // y2 = S1 + CH                          // --
   352  	VPSHUFD $80, XTMP0, XTMP2;           \ // XTMP2 = W[-2] {DDCC}
   353  	;                                    \
   354  	ORL     T1, y3;                      \ // y3 = MAJ = (a|c)&b)|(a&c)             // MAJ
   355  	ADDL    y1, h;                       \ // h = k + w + h + S0                    // --
   356  	ADDL    y2, d;                       \ // d = k + w + h + d + S1 + CH = d + t1  // --
   357  	ADDL    y2, h;                       \ // h = k + w + h + S0 + S1 + CH = t1 + S0// --
   358  	;                                    \
   359  	ADDL    y3, h                        // h = t1 + S0 + MAJ                     // --
   360  
   361  #define ROUND_AND_SCHED_N_3(disp, a, b, c, d, e, f, g, h, XDWORD0, XDWORD1, XDWORD2, XDWORD3) \
   362  	;                                    \ // ################################### RND N + 3 ############################
   363  	;                                    \
   364  	MOVL    a, y3;                       \ // y3 = a						// MAJA
   365  	RORXL   $25, e, y0;                  \ // y0 = e >> 25					// S1A
   366  	RORXL   $11, e, y1;                  \ // y1 = e >> 11					// S1B
   367  	ADDL    (disp + 3*4)(SP)(SRND*1), h; \ // h = k + w + h				// --
   368  	ORL     c, y3;                       \ // y3 = a|c                     // MAJA
   369  	;                                    \
   370  	VPSRLD  $10, XTMP2, XTMP5;           \ // XTMP5 = W[-2] >> 10 {DDCC}
   371  	MOVL    f, y2;                       \ // y2 = f						// CH
   372  	RORXL   $13, a, T1;                  \ // T1 = a >> 13					// S0B
   373  	XORL    y1, y0;                      \ // y0 = (e>>25) ^ (e>>11)		// S1
   374  	XORL    g, y2;                       \ // y2 = f^g						// CH
   375  	;                                    \
   376  	VPSRLQ  $19, XTMP2, XTMP3;           \ // XTMP3 = W[-2] ror 19 {xDxC}
   377  	RORXL   $6, e, y1;                   \ // y1 = (e >> 6)				// S1
   378  	ANDL    e, y2;                       \ // y2 = (f^g)&e					// CH
   379  	ADDL    h, d;                        \ // d = k + w + h + d			// --
   380  	ANDL    b, y3;                       \ // y3 = (a|c)&b					// MAJA
   381  	;                                    \
   382  	VPSRLQ  $17, XTMP2, XTMP2;           \ // XTMP2 = W[-2] ror 17 {xDxC}
   383  	XORL    y1, y0;                      \ // y0 = (e>>25) ^ (e>>11) ^ (e>>6)	// S1
   384  	XORL    g, y2;                       \ // y2 = CH = ((f^g)&e)^g			// CH
   385  	;                                    \
   386  	VPXOR   XTMP3, XTMP2, XTMP2;         \
   387  	RORXL   $22, a, y1;                  \ // y1 = a >> 22					// S0A
   388  	ADDL    y0, y2;                      \ // y2 = S1 + CH					// --
   389  	;                                    \
   390  	VPXOR   XTMP2, XTMP5, XTMP5;         \ // XTMP5 = s1 {xDxC}
   391  	XORL    T1, y1;                      \ // y1 = (a>>22) ^ (a>>13)		// S0
   392  	ADDL    y2, d;                       \ // d = k + w + h + d + S1 + CH = d + t1  // --
   393  	;                                    \
   394  	RORXL   $2, a, T1;                   \ // T1 = (a >> 2)				// S0
   395  	;                                    \
   396  	VPSHUFB shuff_DC00<>(SB), XTMP5, XTMP5;\ // XTMP5 = s1 {DC00}
   397  	;                                    \
   398  	VPADDD  XTMP0, XTMP5, XDWORD0;       \ // XDWORD0 = {W[3], W[2], W[1], W[0]}
   399  	XORL    T1, y1;                      \ // y1 = (a>>22) ^ (a>>13) ^ (a>>2)	// S0
   400  	MOVL    a, T1;                       \ // T1 = a							// MAJB
   401  	ANDL    c, T1;                       \ // T1 = a&c							// MAJB
   402  	ORL     T1, y3;                      \ // y3 = MAJ = (a|c)&b)|(a&c)		// MAJ
   403  	;                                    \
   404  	ADDL    y1, h;                       \ // h = k + w + h + S0				// --
   405  	ADDL    y2, h;                       \ // h = k + w + h + S0 + S1 + CH = t1 + S0// --
   406  	ADDL    y3, h                        // h = t1 + S0 + MAJ				// --
   407  
   408  #define DO_ROUND_N_0(disp, a, b, c, d, e, f, g, h, old_h) \
   409  	;                                  \ // ################################### RND N + 0 ###########################
   410  	MOVL  f, y2;                       \ // y2 = f					// CH
   411  	RORXL $25, e, y0;                  \ // y0 = e >> 25				// S1A
   412  	RORXL $11, e, y1;                  \ // y1 = e >> 11				// S1B
   413  	XORL  g, y2;                       \ // y2 = f^g					// CH
   414  	;                                  \
   415  	XORL  y1, y0;                      \ // y0 = (e>>25) ^ (e>>11)	// S1
   416  	RORXL $6, e, y1;                   \ // y1 = (e >> 6)			// S1
   417  	ANDL  e, y2;                       \ // y2 = (f^g)&e				// CH
   418  	;                                  \
   419  	XORL  y1, y0;                      \ // y0 = (e>>25) ^ (e>>11) ^ (e>>6)	// S1
   420  	RORXL $13, a, T1;                  \ // T1 = a >> 13						// S0B
   421  	XORL  g, y2;                       \ // y2 = CH = ((f^g)&e)^g			// CH
   422  	RORXL $22, a, y1;                  \ // y1 = a >> 22						// S0A
   423  	MOVL  a, y3;                       \ // y3 = a							// MAJA
   424  	;                                  \
   425  	XORL  T1, y1;                      \ // y1 = (a>>22) ^ (a>>13)			// S0
   426  	RORXL $2, a, T1;                   \ // T1 = (a >> 2)					// S0
   427  	ADDL  (disp + 0*4)(SP)(SRND*1), h; \ // h = k + w + h // --
   428  	ORL   c, y3;                       \ // y3 = a|c							// MAJA
   429  	;                                  \
   430  	XORL  T1, y1;                      \ // y1 = (a>>22) ^ (a>>13) ^ (a>>2)	// S0
   431  	MOVL  a, T1;                       \ // T1 = a							// MAJB
   432  	ANDL  b, y3;                       \ // y3 = (a|c)&b						// MAJA
   433  	ANDL  c, T1;                       \ // T1 = a&c							// MAJB
   434  	ADDL  y0, y2;                      \ // y2 = S1 + CH						// --
   435  	;                                  \
   436  	ADDL  h, d;                        \ // d = k + w + h + d					// --
   437  	ORL   T1, y3;                      \ // y3 = MAJ = (a|c)&b)|(a&c)			// MAJ
   438  	ADDL  y1, h;                       \ // h = k + w + h + S0					// --
   439  	ADDL  y2, d                        // d = k + w + h + d + S1 + CH = d + t1	// --
   440  
   441  #define DO_ROUND_N_1(disp, a, b, c, d, e, f, g, h, old_h) \
   442  	;                                  \ // ################################### RND N + 1 ###########################
   443  	ADDL  y2, old_h;                   \ // h = k + w + h + S0 + S1 + CH = t1 + S0 // --
   444  	MOVL  f, y2;                       \ // y2 = f                                // CH
   445  	RORXL $25, e, y0;                  \ // y0 = e >> 25				// S1A
   446  	RORXL $11, e, y1;                  \ // y1 = e >> 11				// S1B
   447  	XORL  g, y2;                       \ // y2 = f^g                             // CH
   448  	;                                  \
   449  	XORL  y1, y0;                      \ // y0 = (e>>25) ^ (e>>11)				// S1
   450  	RORXL $6, e, y1;                   \ // y1 = (e >> 6)						// S1
   451  	ANDL  e, y2;                       \ // y2 = (f^g)&e                         // CH
   452  	ADDL  y3, old_h;                   \ // h = t1 + S0 + MAJ                    // --
   453  	;                                  \
   454  	XORL  y1, y0;                      \ // y0 = (e>>25) ^ (e>>11) ^ (e>>6)		// S1
   455  	RORXL $13, a, T1;                  \ // T1 = a >> 13							// S0B
   456  	XORL  g, y2;                       \ // y2 = CH = ((f^g)&e)^g                // CH
   457  	RORXL $22, a, y1;                  \ // y1 = a >> 22							// S0A
   458  	MOVL  a, y3;                       \ // y3 = a                               // MAJA
   459  	;                                  \
   460  	XORL  T1, y1;                      \ // y1 = (a>>22) ^ (a>>13)				// S0
   461  	RORXL $2, a, T1;                   \ // T1 = (a >> 2)						// S0
   462  	ADDL  (disp + 1*4)(SP)(SRND*1), h; \ // h = k + w + h // --
   463  	ORL   c, y3;                       \ // y3 = a|c                             // MAJA
   464  	;                                  \
   465  	XORL  T1, y1;                      \ // y1 = (a>>22) ^ (a>>13) ^ (a>>2)		// S0
   466  	MOVL  a, T1;                       \ // T1 = a                               // MAJB
   467  	ANDL  b, y3;                       \ // y3 = (a|c)&b                         // MAJA
   468  	ANDL  c, T1;                       \ // T1 = a&c                             // MAJB
   469  	ADDL  y0, y2;                      \ // y2 = S1 + CH                         // --
   470  	;                                  \
   471  	ADDL  h, d;                        \ // d = k + w + h + d                    // --
   472  	ORL   T1, y3;                      \ // y3 = MAJ = (a|c)&b)|(a&c)            // MAJ
   473  	ADDL  y1, h;                       \ // h = k + w + h + S0                   // --
   474  	;                                  \
   475  	ADDL  y2, d                        // d = k + w + h + d + S1 + CH = d + t1 // --
   476  
   477  #define DO_ROUND_N_2(disp, a, b, c, d, e, f, g, h, old_h) \
   478  	;                                  \ // ################################### RND N + 2 ##############################
   479  	ADDL  y2, old_h;                   \ // h = k + w + h + S0 + S1 + CH = t1 + S0// --
   480  	MOVL  f, y2;                       \ // y2 = f								// CH
   481  	RORXL $25, e, y0;                  \ // y0 = e >> 25							// S1A
   482  	RORXL $11, e, y1;                  \ // y1 = e >> 11							// S1B
   483  	XORL  g, y2;                       \ // y2 = f^g								// CH
   484  	;                                  \
   485  	XORL  y1, y0;                      \ // y0 = (e>>25) ^ (e>>11)				// S1
   486  	RORXL $6, e, y1;                   \ // y1 = (e >> 6)						// S1
   487  	ANDL  e, y2;                       \ // y2 = (f^g)&e							// CH
   488  	ADDL  y3, old_h;                   \ // h = t1 + S0 + MAJ					// --
   489  	;                                  \
   490  	XORL  y1, y0;                      \ // y0 = (e>>25) ^ (e>>11) ^ (e>>6)		// S1
   491  	RORXL $13, a, T1;                  \ // T1 = a >> 13							// S0B
   492  	XORL  g, y2;                       \ // y2 = CH = ((f^g)&e)^g                // CH
   493  	RORXL $22, a, y1;                  \ // y1 = a >> 22							// S0A
   494  	MOVL  a, y3;                       \ // y3 = a								// MAJA
   495  	;                                  \
   496  	XORL  T1, y1;                      \ // y1 = (a>>22) ^ (a>>13)				// S0
   497  	RORXL $2, a, T1;                   \ // T1 = (a >> 2)						// S0
   498  	ADDL  (disp + 2*4)(SP)(SRND*1), h; \ // h = k + w + h 	// --
   499  	ORL   c, y3;                       \ // y3 = a|c								// MAJA
   500  	;                                  \
   501  	XORL  T1, y1;                      \ // y1 = (a>>22) ^ (a>>13) ^ (a>>2)		// S0
   502  	MOVL  a, T1;                       \ // T1 = a								// MAJB
   503  	ANDL  b, y3;                       \ // y3 = (a|c)&b							// MAJA
   504  	ANDL  c, T1;                       \ // T1 = a&c								// MAJB
   505  	ADDL  y0, y2;                      \ // y2 = S1 + CH							// --
   506  	;                                  \
   507  	ADDL  h, d;                        \ // d = k + w + h + d					// --
   508  	ORL   T1, y3;                      \ // y3 = MAJ = (a|c)&b)|(a&c)			// MAJ
   509  	ADDL  y1, h;                       \ // h = k + w + h + S0					// --
   510  	;                                  \
   511  	ADDL  y2, d                        // d = k + w + h + d + S1 + CH = d + t1 // --
   512  
   513  #define DO_ROUND_N_3(disp, a, b, c, d, e, f, g, h, old_h) \
   514  	;                                  \ // ################################### RND N + 3 ###########################
   515  	ADDL  y2, old_h;                   \ // h = k + w + h + S0 + S1 + CH = t1 + S0// --
   516  	MOVL  f, y2;                       \ // y2 = f								// CH
   517  	RORXL $25, e, y0;                  \ // y0 = e >> 25							// S1A
   518  	RORXL $11, e, y1;                  \ // y1 = e >> 11							// S1B
   519  	XORL  g, y2;                       \ // y2 = f^g								// CH
   520  	;                                  \
   521  	XORL  y1, y0;                      \ // y0 = (e>>25) ^ (e>>11)				// S1
   522  	RORXL $6, e, y1;                   \ // y1 = (e >> 6)						// S1
   523  	ANDL  e, y2;                       \ // y2 = (f^g)&e							// CH
   524  	ADDL  y3, old_h;                   \ // h = t1 + S0 + MAJ					// --
   525  	;                                  \
   526  	XORL  y1, y0;                      \ // y0 = (e>>25) ^ (e>>11) ^ (e>>6)		// S1
   527  	RORXL $13, a, T1;                  \ // T1 = a >> 13							// S0B
   528  	XORL  g, y2;                       \ // y2 = CH = ((f^g)&e)^g				// CH
   529  	RORXL $22, a, y1;                  \ // y1 = a >> 22							// S0A
   530  	MOVL  a, y3;                       \ // y3 = a								// MAJA
   531  	;                                  \
   532  	XORL  T1, y1;                      \ // y1 = (a>>22) ^ (a>>13)				// S0
   533  	RORXL $2, a, T1;                   \ // T1 = (a >> 2)						// S0
   534  	ADDL  (disp + 3*4)(SP)(SRND*1), h; \ // h = k + w + h 	// --
   535  	ORL   c, y3;                       \ // y3 = a|c								// MAJA
   536  	;                                  \
   537  	XORL  T1, y1;                      \ // y1 = (a>>22) ^ (a>>13) ^ (a>>2)		// S0
   538  	MOVL  a, T1;                       \ // T1 = a								// MAJB
   539  	ANDL  b, y3;                       \ // y3 = (a|c)&b							// MAJA
   540  	ANDL  c, T1;                       \ // T1 = a&c								// MAJB
   541  	ADDL  y0, y2;                      \ // y2 = S1 + CH							// --
   542  	;                                  \
   543  	ADDL  h, d;                        \ // d = k + w + h + d					// --
   544  	ORL   T1, y3;                      \ // y3 = MAJ = (a|c)&b)|(a&c)			// MAJ
   545  	ADDL  y1, h;                       \ // h = k + w + h + S0					// --
   546  	;                                  \
   547  	ADDL  y2, d;                       \ // d = k + w + h + d + S1 + CH = d + t1	// --
   548  	;                                  \
   549  	ADDL  y2, h;                       \ // h = k + w + h + S0 + S1 + CH = t1 + S0// --
   550  	;                                  \
   551  	ADDL  y3, h                        // h = t1 + S0 + MAJ					// --
   552  
   553  TEXT ·block(SB), 0, $536-32
   554  	CMPB ·useAVX2(SB), $1
   555  	JE   avx2
   556  
   557  	MOVQ p_base+8(FP), SI
   558  	MOVQ p_len+16(FP), DX
   559  	SHRQ $6, DX
   560  	SHLQ $6, DX
   561  
   562  	LEAQ (SI)(DX*1), DI
   563  	MOVQ DI, 256(SP)
   564  	CMPQ SI, DI
   565  	JEQ  end
   566  
   567  	MOVQ dig+0(FP), BP
   568  	MOVL (0*4)(BP), R8  // a = H0
   569  	MOVL (1*4)(BP), R9  // b = H1
   570  	MOVL (2*4)(BP), R10 // c = H2
   571  	MOVL (3*4)(BP), R11 // d = H3
   572  	MOVL (4*4)(BP), R12 // e = H4
   573  	MOVL (5*4)(BP), R13 // f = H5
   574  	MOVL (6*4)(BP), R14 // g = H6
   575  	MOVL (7*4)(BP), R15 // h = H7
   576  
   577  loop:
   578  	MOVQ SP, BP
   579  
   580  	SHA256ROUND0(0, 0x428a2f98, R8, R9, R10, R11, R12, R13, R14, R15)
   581  	SHA256ROUND0(1, 0x71374491, R15, R8, R9, R10, R11, R12, R13, R14)
   582  	SHA256ROUND0(2, 0xb5c0fbcf, R14, R15, R8, R9, R10, R11, R12, R13)
   583  	SHA256ROUND0(3, 0xe9b5dba5, R13, R14, R15, R8, R9, R10, R11, R12)
   584  	SHA256ROUND0(4, 0x3956c25b, R12, R13, R14, R15, R8, R9, R10, R11)
   585  	SHA256ROUND0(5, 0x59f111f1, R11, R12, R13, R14, R15, R8, R9, R10)
   586  	SHA256ROUND0(6, 0x923f82a4, R10, R11, R12, R13, R14, R15, R8, R9)
   587  	SHA256ROUND0(7, 0xab1c5ed5, R9, R10, R11, R12, R13, R14, R15, R8)
   588  	SHA256ROUND0(8, 0xd807aa98, R8, R9, R10, R11, R12, R13, R14, R15)
   589  	SHA256ROUND0(9, 0x12835b01, R15, R8, R9, R10, R11, R12, R13, R14)
   590  	SHA256ROUND0(10, 0x243185be, R14, R15, R8, R9, R10, R11, R12, R13)
   591  	SHA256ROUND0(11, 0x550c7dc3, R13, R14, R15, R8, R9, R10, R11, R12)
   592  	SHA256ROUND0(12, 0x72be5d74, R12, R13, R14, R15, R8, R9, R10, R11)
   593  	SHA256ROUND0(13, 0x80deb1fe, R11, R12, R13, R14, R15, R8, R9, R10)
   594  	SHA256ROUND0(14, 0x9bdc06a7, R10, R11, R12, R13, R14, R15, R8, R9)
   595  	SHA256ROUND0(15, 0xc19bf174, R9, R10, R11, R12, R13, R14, R15, R8)
   596  
   597  	SHA256ROUND1(16, 0xe49b69c1, R8, R9, R10, R11, R12, R13, R14, R15)
   598  	SHA256ROUND1(17, 0xefbe4786, R15, R8, R9, R10, R11, R12, R13, R14)
   599  	SHA256ROUND1(18, 0x0fc19dc6, R14, R15, R8, R9, R10, R11, R12, R13)
   600  	SHA256ROUND1(19, 0x240ca1cc, R13, R14, R15, R8, R9, R10, R11, R12)
   601  	SHA256ROUND1(20, 0x2de92c6f, R12, R13, R14, R15, R8, R9, R10, R11)
   602  	SHA256ROUND1(21, 0x4a7484aa, R11, R12, R13, R14, R15, R8, R9, R10)
   603  	SHA256ROUND1(22, 0x5cb0a9dc, R10, R11, R12, R13, R14, R15, R8, R9)
   604  	SHA256ROUND1(23, 0x76f988da, R9, R10, R11, R12, R13, R14, R15, R8)
   605  	SHA256ROUND1(24, 0x983e5152, R8, R9, R10, R11, R12, R13, R14, R15)
   606  	SHA256ROUND1(25, 0xa831c66d, R15, R8, R9, R10, R11, R12, R13, R14)
   607  	SHA256ROUND1(26, 0xb00327c8, R14, R15, R8, R9, R10, R11, R12, R13)
   608  	SHA256ROUND1(27, 0xbf597fc7, R13, R14, R15, R8, R9, R10, R11, R12)
   609  	SHA256ROUND1(28, 0xc6e00bf3, R12, R13, R14, R15, R8, R9, R10, R11)
   610  	SHA256ROUND1(29, 0xd5a79147, R11, R12, R13, R14, R15, R8, R9, R10)
   611  	SHA256ROUND1(30, 0x06ca6351, R10, R11, R12, R13, R14, R15, R8, R9)
   612  	SHA256ROUND1(31, 0x14292967, R9, R10, R11, R12, R13, R14, R15, R8)
   613  	SHA256ROUND1(32, 0x27b70a85, R8, R9, R10, R11, R12, R13, R14, R15)
   614  	SHA256ROUND1(33, 0x2e1b2138, R15, R8, R9, R10, R11, R12, R13, R14)
   615  	SHA256ROUND1(34, 0x4d2c6dfc, R14, R15, R8, R9, R10, R11, R12, R13)
   616  	SHA256ROUND1(35, 0x53380d13, R13, R14, R15, R8, R9, R10, R11, R12)
   617  	SHA256ROUND1(36, 0x650a7354, R12, R13, R14, R15, R8, R9, R10, R11)
   618  	SHA256ROUND1(37, 0x766a0abb, R11, R12, R13, R14, R15, R8, R9, R10)
   619  	SHA256ROUND1(38, 0x81c2c92e, R10, R11, R12, R13, R14, R15, R8, R9)
   620  	SHA256ROUND1(39, 0x92722c85, R9, R10, R11, R12, R13, R14, R15, R8)
   621  	SHA256ROUND1(40, 0xa2bfe8a1, R8, R9, R10, R11, R12, R13, R14, R15)
   622  	SHA256ROUND1(41, 0xa81a664b, R15, R8, R9, R10, R11, R12, R13, R14)
   623  	SHA256ROUND1(42, 0xc24b8b70, R14, R15, R8, R9, R10, R11, R12, R13)
   624  	SHA256ROUND1(43, 0xc76c51a3, R13, R14, R15, R8, R9, R10, R11, R12)
   625  	SHA256ROUND1(44, 0xd192e819, R12, R13, R14, R15, R8, R9, R10, R11)
   626  	SHA256ROUND1(45, 0xd6990624, R11, R12, R13, R14, R15, R8, R9, R10)
   627  	SHA256ROUND1(46, 0xf40e3585, R10, R11, R12, R13, R14, R15, R8, R9)
   628  	SHA256ROUND1(47, 0x106aa070, R9, R10, R11, R12, R13, R14, R15, R8)
   629  	SHA256ROUND1(48, 0x19a4c116, R8, R9, R10, R11, R12, R13, R14, R15)
   630  	SHA256ROUND1(49, 0x1e376c08, R15, R8, R9, R10, R11, R12, R13, R14)
   631  	SHA256ROUND1(50, 0x2748774c, R14, R15, R8, R9, R10, R11, R12, R13)
   632  	SHA256ROUND1(51, 0x34b0bcb5, R13, R14, R15, R8, R9, R10, R11, R12)
   633  	SHA256ROUND1(52, 0x391c0cb3, R12, R13, R14, R15, R8, R9, R10, R11)
   634  	SHA256ROUND1(53, 0x4ed8aa4a, R11, R12, R13, R14, R15, R8, R9, R10)
   635  	SHA256ROUND1(54, 0x5b9cca4f, R10, R11, R12, R13, R14, R15, R8, R9)
   636  	SHA256ROUND1(55, 0x682e6ff3, R9, R10, R11, R12, R13, R14, R15, R8)
   637  	SHA256ROUND1(56, 0x748f82ee, R8, R9, R10, R11, R12, R13, R14, R15)
   638  	SHA256ROUND1(57, 0x78a5636f, R15, R8, R9, R10, R11, R12, R13, R14)
   639  	SHA256ROUND1(58, 0x84c87814, R14, R15, R8, R9, R10, R11, R12, R13)
   640  	SHA256ROUND1(59, 0x8cc70208, R13, R14, R15, R8, R9, R10, R11, R12)
   641  	SHA256ROUND1(60, 0x90befffa, R12, R13, R14, R15, R8, R9, R10, R11)
   642  	SHA256ROUND1(61, 0xa4506ceb, R11, R12, R13, R14, R15, R8, R9, R10)
   643  	SHA256ROUND1(62, 0xbef9a3f7, R10, R11, R12, R13, R14, R15, R8, R9)
   644  	SHA256ROUND1(63, 0xc67178f2, R9, R10, R11, R12, R13, R14, R15, R8)
   645  
   646  	MOVQ dig+0(FP), BP
   647  	ADDL (0*4)(BP), R8  // H0 = a + H0
   648  	MOVL R8, (0*4)(BP)
   649  	ADDL (1*4)(BP), R9  // H1 = b + H1
   650  	MOVL R9, (1*4)(BP)
   651  	ADDL (2*4)(BP), R10 // H2 = c + H2
   652  	MOVL R10, (2*4)(BP)
   653  	ADDL (3*4)(BP), R11 // H3 = d + H3
   654  	MOVL R11, (3*4)(BP)
   655  	ADDL (4*4)(BP), R12 // H4 = e + H4
   656  	MOVL R12, (4*4)(BP)
   657  	ADDL (5*4)(BP), R13 // H5 = f + H5
   658  	MOVL R13, (5*4)(BP)
   659  	ADDL (6*4)(BP), R14 // H6 = g + H6
   660  	MOVL R14, (6*4)(BP)
   661  	ADDL (7*4)(BP), R15 // H7 = h + H7
   662  	MOVL R15, (7*4)(BP)
   663  
   664  	ADDQ $64, SI
   665  	CMPQ SI, 256(SP)
   666  	JB   loop
   667  
   668  end:
   669  	RET
   670  
   671  avx2:
   672  	MOVQ dig+0(FP), CTX          // d.h[8]
   673  	MOVQ p_base+8(FP), INP
   674  	MOVQ p_len+16(FP), NUM_BYTES
   675  
   676  	LEAQ -64(INP)(NUM_BYTES*1), NUM_BYTES // Pointer to the last block
   677  	MOVQ NUM_BYTES, _INP_END(SP)
   678  
   679  	CMPQ NUM_BYTES, INP
   680  	JE   avx2_only_one_block
   681  
   682  	// Load initial digest
   683  	MOVL 0(CTX), a  // a = H0
   684  	MOVL 4(CTX), b  // b = H1
   685  	MOVL 8(CTX), c  // c = H2
   686  	MOVL 12(CTX), d // d = H3
   687  	MOVL 16(CTX), e // e = H4
   688  	MOVL 20(CTX), f // f = H5
   689  	MOVL 24(CTX), g // g = H6
   690  	MOVL 28(CTX), h // h = H7
   691  
   692  avx2_loop0: // at each iteration works with one block (512 bit)
   693  
   694  	VMOVDQU (0*32)(INP), XTMP0
   695  	VMOVDQU (1*32)(INP), XTMP1
   696  	VMOVDQU (2*32)(INP), XTMP2
   697  	VMOVDQU (3*32)(INP), XTMP3
   698  
   699  	VMOVDQU flip_mask<>(SB), BYTE_FLIP_MASK
   700  
   701  	// Apply Byte Flip Mask: LE -> BE
   702  	VPSHUFB BYTE_FLIP_MASK, XTMP0, XTMP0
   703  	VPSHUFB BYTE_FLIP_MASK, XTMP1, XTMP1
   704  	VPSHUFB BYTE_FLIP_MASK, XTMP2, XTMP2
   705  	VPSHUFB BYTE_FLIP_MASK, XTMP3, XTMP3
   706  
   707  	// Transpose data into high/low parts
   708  	VPERM2I128 $0x20, XTMP2, XTMP0, XDWORD0 // w3, w2, w1, w0
   709  	VPERM2I128 $0x31, XTMP2, XTMP0, XDWORD1 // w7, w6, w5, w4
   710  	VPERM2I128 $0x20, XTMP3, XTMP1, XDWORD2 // w11, w10, w9, w8
   711  	VPERM2I128 $0x31, XTMP3, XTMP1, XDWORD3 // w15, w14, w13, w12
   712  
   713  	MOVQ $K256<>(SB), TBL // Loading address of table with round-specific constants
   714  
   715  avx2_last_block_enter:
   716  	ADDQ $64, INP
   717  	MOVQ INP, _INP(SP)
   718  	XORQ SRND, SRND
   719  
   720  avx2_loop1: // for w0 - w47
   721  	// Do 4 rounds and scheduling
   722  	VPADDD  0*32(TBL)(SRND*1), XDWORD0, XFER
   723  	VMOVDQU XFER, (_XFER + 0*32)(SP)(SRND*1)
   724  	ROUND_AND_SCHED_N_0(_XFER + 0*32, a, b, c, d, e, f, g, h, XDWORD0, XDWORD1, XDWORD2, XDWORD3)
   725  	ROUND_AND_SCHED_N_1(_XFER + 0*32, h, a, b, c, d, e, f, g, XDWORD0, XDWORD1, XDWORD2, XDWORD3)
   726  	ROUND_AND_SCHED_N_2(_XFER + 0*32, g, h, a, b, c, d, e, f, XDWORD0, XDWORD1, XDWORD2, XDWORD3)
   727  	ROUND_AND_SCHED_N_3(_XFER + 0*32, f, g, h, a, b, c, d, e, XDWORD0, XDWORD1, XDWORD2, XDWORD3)
   728  
   729  	// Do 4 rounds and scheduling
   730  	VPADDD  1*32(TBL)(SRND*1), XDWORD1, XFER
   731  	VMOVDQU XFER, (_XFER + 1*32)(SP)(SRND*1)
   732  	ROUND_AND_SCHED_N_0(_XFER + 1*32, e, f, g, h, a, b, c, d, XDWORD1, XDWORD2, XDWORD3, XDWORD0)
   733  	ROUND_AND_SCHED_N_1(_XFER + 1*32, d, e, f, g, h, a, b, c, XDWORD1, XDWORD2, XDWORD3, XDWORD0)
   734  	ROUND_AND_SCHED_N_2(_XFER + 1*32, c, d, e, f, g, h, a, b, XDWORD1, XDWORD2, XDWORD3, XDWORD0)
   735  	ROUND_AND_SCHED_N_3(_XFER + 1*32, b, c, d, e, f, g, h, a, XDWORD1, XDWORD2, XDWORD3, XDWORD0)
   736  
   737  	// Do 4 rounds and scheduling
   738  	VPADDD  2*32(TBL)(SRND*1), XDWORD2, XFER
   739  	VMOVDQU XFER, (_XFER + 2*32)(SP)(SRND*1)
   740  	ROUND_AND_SCHED_N_0(_XFER + 2*32, a, b, c, d, e, f, g, h, XDWORD2, XDWORD3, XDWORD0, XDWORD1)
   741  	ROUND_AND_SCHED_N_1(_XFER + 2*32, h, a, b, c, d, e, f, g, XDWORD2, XDWORD3, XDWORD0, XDWORD1)
   742  	ROUND_AND_SCHED_N_2(_XFER + 2*32, g, h, a, b, c, d, e, f, XDWORD2, XDWORD3, XDWORD0, XDWORD1)
   743  	ROUND_AND_SCHED_N_3(_XFER + 2*32, f, g, h, a, b, c, d, e, XDWORD2, XDWORD3, XDWORD0, XDWORD1)
   744  
   745  	// Do 4 rounds and scheduling
   746  	VPADDD  3*32(TBL)(SRND*1), XDWORD3, XFER
   747  	VMOVDQU XFER, (_XFER + 3*32)(SP)(SRND*1)
   748  	ROUND_AND_SCHED_N_0(_XFER + 3*32, e, f, g, h, a, b, c, d, XDWORD3, XDWORD0, XDWORD1, XDWORD2)
   749  	ROUND_AND_SCHED_N_1(_XFER + 3*32, d, e, f, g, h, a, b, c, XDWORD3, XDWORD0, XDWORD1, XDWORD2)
   750  	ROUND_AND_SCHED_N_2(_XFER + 3*32, c, d, e, f, g, h, a, b, XDWORD3, XDWORD0, XDWORD1, XDWORD2)
   751  	ROUND_AND_SCHED_N_3(_XFER + 3*32, b, c, d, e, f, g, h, a, XDWORD3, XDWORD0, XDWORD1, XDWORD2)
   752  
   753  	ADDQ $4*32, SRND
   754  	CMPQ SRND, $3*4*32
   755  	JB   avx2_loop1
   756  
   757  avx2_loop2:
   758  	// w48 - w63 processed with no scheduling (last 16 rounds)
   759  	VPADDD  0*32(TBL)(SRND*1), XDWORD0, XFER
   760  	VMOVDQU XFER, (_XFER + 0*32)(SP)(SRND*1)
   761  	DO_ROUND_N_0(_XFER + 0*32, a, b, c, d, e, f, g, h, h)
   762  	DO_ROUND_N_1(_XFER + 0*32, h, a, b, c, d, e, f, g, h)
   763  	DO_ROUND_N_2(_XFER + 0*32, g, h, a, b, c, d, e, f, g)
   764  	DO_ROUND_N_3(_XFER + 0*32, f, g, h, a, b, c, d, e, f)
   765  
   766  	VPADDD  1*32(TBL)(SRND*1), XDWORD1, XFER
   767  	VMOVDQU XFER, (_XFER + 1*32)(SP)(SRND*1)
   768  	DO_ROUND_N_0(_XFER + 1*32, e, f, g, h, a, b, c, d, e)
   769  	DO_ROUND_N_1(_XFER + 1*32, d, e, f, g, h, a, b, c, d)
   770  	DO_ROUND_N_2(_XFER + 1*32, c, d, e, f, g, h, a, b, c)
   771  	DO_ROUND_N_3(_XFER + 1*32, b, c, d, e, f, g, h, a, b)
   772  
   773  	ADDQ $2*32, SRND
   774  
   775  	VMOVDQU XDWORD2, XDWORD0
   776  	VMOVDQU XDWORD3, XDWORD1
   777  
   778  	CMPQ SRND, $4*4*32
   779  	JB   avx2_loop2
   780  
   781  	MOVQ dig+0(FP), CTX // d.h[8]
   782  	MOVQ _INP(SP), INP
   783  
   784  	addm(  0(CTX), a)
   785  	addm(  4(CTX), b)
   786  	addm(  8(CTX), c)
   787  	addm( 12(CTX), d)
   788  	addm( 16(CTX), e)
   789  	addm( 20(CTX), f)
   790  	addm( 24(CTX), g)
   791  	addm( 28(CTX), h)
   792  
   793  	CMPQ _INP_END(SP), INP
   794  	JB   done_hash
   795  
   796  	XORQ SRND, SRND
   797  
   798  avx2_loop3: // Do second block using previously scheduled results
   799  	DO_ROUND_N_0(_XFER + 0*32 + 16, a, b, c, d, e, f, g, h, a)
   800  	DO_ROUND_N_1(_XFER + 0*32 + 16, h, a, b, c, d, e, f, g, h)
   801  	DO_ROUND_N_2(_XFER + 0*32 + 16, g, h, a, b, c, d, e, f, g)
   802  	DO_ROUND_N_3(_XFER + 0*32 + 16, f, g, h, a, b, c, d, e, f)
   803  
   804  	DO_ROUND_N_0(_XFER + 1*32 + 16, e, f, g, h, a, b, c, d, e)
   805  	DO_ROUND_N_1(_XFER + 1*32 + 16, d, e, f, g, h, a, b, c, d)
   806  	DO_ROUND_N_2(_XFER + 1*32 + 16, c, d, e, f, g, h, a, b, c)
   807  	DO_ROUND_N_3(_XFER + 1*32 + 16, b, c, d, e, f, g, h, a, b)
   808  
   809  	ADDQ $2*32, SRND
   810  	CMPQ SRND, $4*4*32
   811  	JB   avx2_loop3
   812  
   813  	MOVQ dig+0(FP), CTX // d.h[8]
   814  	MOVQ _INP(SP), INP
   815  	ADDQ $64, INP
   816  
   817  	addm(  0(CTX), a)
   818  	addm(  4(CTX), b)
   819  	addm(  8(CTX), c)
   820  	addm( 12(CTX), d)
   821  	addm( 16(CTX), e)
   822  	addm( 20(CTX), f)
   823  	addm( 24(CTX), g)
   824  	addm( 28(CTX), h)
   825  
   826  	CMPQ _INP_END(SP), INP
   827  	JA   avx2_loop0
   828  	JB   done_hash
   829  
   830  avx2_do_last_block:
   831  
   832  	VMOVDQU 0(INP), XWORD0
   833  	VMOVDQU 16(INP), XWORD1
   834  	VMOVDQU 32(INP), XWORD2
   835  	VMOVDQU 48(INP), XWORD3
   836  
   837  	VMOVDQU flip_mask<>(SB), BYTE_FLIP_MASK
   838  
   839  	VPSHUFB X_BYTE_FLIP_MASK, XWORD0, XWORD0
   840  	VPSHUFB X_BYTE_FLIP_MASK, XWORD1, XWORD1
   841  	VPSHUFB X_BYTE_FLIP_MASK, XWORD2, XWORD2
   842  	VPSHUFB X_BYTE_FLIP_MASK, XWORD3, XWORD3
   843  
   844  	MOVQ $K256<>(SB), TBL
   845  
   846  	JMP avx2_last_block_enter
   847  
   848  avx2_only_one_block:
   849  	// Load initial digest
   850  	MOVL 0(CTX), a  // a = H0
   851  	MOVL 4(CTX), b  // b = H1
   852  	MOVL 8(CTX), c  // c = H2
   853  	MOVL 12(CTX), d // d = H3
   854  	MOVL 16(CTX), e // e = H4
   855  	MOVL 20(CTX), f // f = H5
   856  	MOVL 24(CTX), g // g = H6
   857  	MOVL 28(CTX), h // h = H7
   858  
   859  	JMP avx2_do_last_block
   860  
   861  done_hash:
   862  	VZEROUPPER
   863  	RET
   864  
   865  // shuffle byte order from LE to BE
   866  DATA flip_mask<>+0x00(SB)/8, $0x0405060700010203
   867  DATA flip_mask<>+0x08(SB)/8, $0x0c0d0e0f08090a0b
   868  DATA flip_mask<>+0x10(SB)/8, $0x0405060700010203
   869  DATA flip_mask<>+0x18(SB)/8, $0x0c0d0e0f08090a0b
   870  GLOBL flip_mask<>(SB), 8, $32
   871  
   872  // shuffle xBxA -> 00BA
   873  DATA shuff_00BA<>+0x00(SB)/8, $0x0b0a090803020100
   874  DATA shuff_00BA<>+0x08(SB)/8, $0xFFFFFFFFFFFFFFFF
   875  DATA shuff_00BA<>+0x10(SB)/8, $0x0b0a090803020100
   876  DATA shuff_00BA<>+0x18(SB)/8, $0xFFFFFFFFFFFFFFFF
   877  GLOBL shuff_00BA<>(SB), 8, $32
   878  
   879  // shuffle xDxC -> DC00
   880  DATA shuff_DC00<>+0x00(SB)/8, $0xFFFFFFFFFFFFFFFF
   881  DATA shuff_DC00<>+0x08(SB)/8, $0x0b0a090803020100
   882  DATA shuff_DC00<>+0x10(SB)/8, $0xFFFFFFFFFFFFFFFF
   883  DATA shuff_DC00<>+0x18(SB)/8, $0x0b0a090803020100
   884  GLOBL shuff_DC00<>(SB), 8, $32
   885  
   886  // Round specific constants
   887  DATA K256<>+0x00(SB)/4, $0x428a2f98 // k1
   888  DATA K256<>+0x04(SB)/4, $0x71374491 // k2
   889  DATA K256<>+0x08(SB)/4, $0xb5c0fbcf // k3
   890  DATA K256<>+0x0c(SB)/4, $0xe9b5dba5 // k4
   891  DATA K256<>+0x10(SB)/4, $0x428a2f98 // k1
   892  DATA K256<>+0x14(SB)/4, $0x71374491 // k2
   893  DATA K256<>+0x18(SB)/4, $0xb5c0fbcf // k3
   894  DATA K256<>+0x1c(SB)/4, $0xe9b5dba5 // k4
   895  
   896  DATA K256<>+0x20(SB)/4, $0x3956c25b // k5 - k8
   897  DATA K256<>+0x24(SB)/4, $0x59f111f1
   898  DATA K256<>+0x28(SB)/4, $0x923f82a4
   899  DATA K256<>+0x2c(SB)/4, $0xab1c5ed5
   900  DATA K256<>+0x30(SB)/4, $0x3956c25b
   901  DATA K256<>+0x34(SB)/4, $0x59f111f1
   902  DATA K256<>+0x38(SB)/4, $0x923f82a4
   903  DATA K256<>+0x3c(SB)/4, $0xab1c5ed5
   904  
   905  DATA K256<>+0x40(SB)/4, $0xd807aa98 // k9 - k12
   906  DATA K256<>+0x44(SB)/4, $0x12835b01
   907  DATA K256<>+0x48(SB)/4, $0x243185be
   908  DATA K256<>+0x4c(SB)/4, $0x550c7dc3
   909  DATA K256<>+0x50(SB)/4, $0xd807aa98
   910  DATA K256<>+0x54(SB)/4, $0x12835b01
   911  DATA K256<>+0x58(SB)/4, $0x243185be
   912  DATA K256<>+0x5c(SB)/4, $0x550c7dc3
   913  
   914  DATA K256<>+0x60(SB)/4, $0x72be5d74 // k13 - k16
   915  DATA K256<>+0x64(SB)/4, $0x80deb1fe
   916  DATA K256<>+0x68(SB)/4, $0x9bdc06a7
   917  DATA K256<>+0x6c(SB)/4, $0xc19bf174
   918  DATA K256<>+0x70(SB)/4, $0x72be5d74
   919  DATA K256<>+0x74(SB)/4, $0x80deb1fe
   920  DATA K256<>+0x78(SB)/4, $0x9bdc06a7
   921  DATA K256<>+0x7c(SB)/4, $0xc19bf174
   922  
   923  DATA K256<>+0x80(SB)/4, $0xe49b69c1 // k17 - k20
   924  DATA K256<>+0x84(SB)/4, $0xefbe4786
   925  DATA K256<>+0x88(SB)/4, $0x0fc19dc6
   926  DATA K256<>+0x8c(SB)/4, $0x240ca1cc
   927  DATA K256<>+0x90(SB)/4, $0xe49b69c1
   928  DATA K256<>+0x94(SB)/4, $0xefbe4786
   929  DATA K256<>+0x98(SB)/4, $0x0fc19dc6
   930  DATA K256<>+0x9c(SB)/4, $0x240ca1cc
   931  
   932  DATA K256<>+0xa0(SB)/4, $0x2de92c6f // k21 - k24
   933  DATA K256<>+0xa4(SB)/4, $0x4a7484aa
   934  DATA K256<>+0xa8(SB)/4, $0x5cb0a9dc
   935  DATA K256<>+0xac(SB)/4, $0x76f988da
   936  DATA K256<>+0xb0(SB)/4, $0x2de92c6f
   937  DATA K256<>+0xb4(SB)/4, $0x4a7484aa
   938  DATA K256<>+0xb8(SB)/4, $0x5cb0a9dc
   939  DATA K256<>+0xbc(SB)/4, $0x76f988da
   940  
   941  DATA K256<>+0xc0(SB)/4, $0x983e5152 // k25 - k28
   942  DATA K256<>+0xc4(SB)/4, $0xa831c66d
   943  DATA K256<>+0xc8(SB)/4, $0xb00327c8
   944  DATA K256<>+0xcc(SB)/4, $0xbf597fc7
   945  DATA K256<>+0xd0(SB)/4, $0x983e5152
   946  DATA K256<>+0xd4(SB)/4, $0xa831c66d
   947  DATA K256<>+0xd8(SB)/4, $0xb00327c8
   948  DATA K256<>+0xdc(SB)/4, $0xbf597fc7
   949  
   950  DATA K256<>+0xe0(SB)/4, $0xc6e00bf3 // k29 - k32
   951  DATA K256<>+0xe4(SB)/4, $0xd5a79147
   952  DATA K256<>+0xe8(SB)/4, $0x06ca6351
   953  DATA K256<>+0xec(SB)/4, $0x14292967
   954  DATA K256<>+0xf0(SB)/4, $0xc6e00bf3
   955  DATA K256<>+0xf4(SB)/4, $0xd5a79147
   956  DATA K256<>+0xf8(SB)/4, $0x06ca6351
   957  DATA K256<>+0xfc(SB)/4, $0x14292967
   958  
   959  DATA K256<>+0x100(SB)/4, $0x27b70a85
   960  DATA K256<>+0x104(SB)/4, $0x2e1b2138
   961  DATA K256<>+0x108(SB)/4, $0x4d2c6dfc
   962  DATA K256<>+0x10c(SB)/4, $0x53380d13
   963  DATA K256<>+0x110(SB)/4, $0x27b70a85
   964  DATA K256<>+0x114(SB)/4, $0x2e1b2138
   965  DATA K256<>+0x118(SB)/4, $0x4d2c6dfc
   966  DATA K256<>+0x11c(SB)/4, $0x53380d13
   967  
   968  DATA K256<>+0x120(SB)/4, $0x650a7354
   969  DATA K256<>+0x124(SB)/4, $0x766a0abb
   970  DATA K256<>+0x128(SB)/4, $0x81c2c92e
   971  DATA K256<>+0x12c(SB)/4, $0x92722c85
   972  DATA K256<>+0x130(SB)/4, $0x650a7354
   973  DATA K256<>+0x134(SB)/4, $0x766a0abb
   974  DATA K256<>+0x138(SB)/4, $0x81c2c92e
   975  DATA K256<>+0x13c(SB)/4, $0x92722c85
   976  
   977  DATA K256<>+0x140(SB)/4, $0xa2bfe8a1
   978  DATA K256<>+0x144(SB)/4, $0xa81a664b
   979  DATA K256<>+0x148(SB)/4, $0xc24b8b70
   980  DATA K256<>+0x14c(SB)/4, $0xc76c51a3
   981  DATA K256<>+0x150(SB)/4, $0xa2bfe8a1
   982  DATA K256<>+0x154(SB)/4, $0xa81a664b
   983  DATA K256<>+0x158(SB)/4, $0xc24b8b70
   984  DATA K256<>+0x15c(SB)/4, $0xc76c51a3
   985  
   986  DATA K256<>+0x160(SB)/4, $0xd192e819
   987  DATA K256<>+0x164(SB)/4, $0xd6990624
   988  DATA K256<>+0x168(SB)/4, $0xf40e3585
   989  DATA K256<>+0x16c(SB)/4, $0x106aa070
   990  DATA K256<>+0x170(SB)/4, $0xd192e819
   991  DATA K256<>+0x174(SB)/4, $0xd6990624
   992  DATA K256<>+0x178(SB)/4, $0xf40e3585
   993  DATA K256<>+0x17c(SB)/4, $0x106aa070
   994  
   995  DATA K256<>+0x180(SB)/4, $0x19a4c116
   996  DATA K256<>+0x184(SB)/4, $0x1e376c08
   997  DATA K256<>+0x188(SB)/4, $0x2748774c
   998  DATA K256<>+0x18c(SB)/4, $0x34b0bcb5
   999  DATA K256<>+0x190(SB)/4, $0x19a4c116
  1000  DATA K256<>+0x194(SB)/4, $0x1e376c08
  1001  DATA K256<>+0x198(SB)/4, $0x2748774c
  1002  DATA K256<>+0x19c(SB)/4, $0x34b0bcb5
  1003  
  1004  DATA K256<>+0x1a0(SB)/4, $0x391c0cb3
  1005  DATA K256<>+0x1a4(SB)/4, $0x4ed8aa4a
  1006  DATA K256<>+0x1a8(SB)/4, $0x5b9cca4f
  1007  DATA K256<>+0x1ac(SB)/4, $0x682e6ff3
  1008  DATA K256<>+0x1b0(SB)/4, $0x391c0cb3
  1009  DATA K256<>+0x1b4(SB)/4, $0x4ed8aa4a
  1010  DATA K256<>+0x1b8(SB)/4, $0x5b9cca4f
  1011  DATA K256<>+0x1bc(SB)/4, $0x682e6ff3
  1012  
  1013  DATA K256<>+0x1c0(SB)/4, $0x748f82ee
  1014  DATA K256<>+0x1c4(SB)/4, $0x78a5636f
  1015  DATA K256<>+0x1c8(SB)/4, $0x84c87814
  1016  DATA K256<>+0x1cc(SB)/4, $0x8cc70208
  1017  DATA K256<>+0x1d0(SB)/4, $0x748f82ee
  1018  DATA K256<>+0x1d4(SB)/4, $0x78a5636f
  1019  DATA K256<>+0x1d8(SB)/4, $0x84c87814
  1020  DATA K256<>+0x1dc(SB)/4, $0x8cc70208
  1021  
  1022  DATA K256<>+0x1e0(SB)/4, $0x90befffa
  1023  DATA K256<>+0x1e4(SB)/4, $0xa4506ceb
  1024  DATA K256<>+0x1e8(SB)/4, $0xbef9a3f7
  1025  DATA K256<>+0x1ec(SB)/4, $0xc67178f2
  1026  DATA K256<>+0x1f0(SB)/4, $0x90befffa
  1027  DATA K256<>+0x1f4(SB)/4, $0xa4506ceb
  1028  DATA K256<>+0x1f8(SB)/4, $0xbef9a3f7
  1029  DATA K256<>+0x1fc(SB)/4, $0xc67178f2
  1030  
  1031  GLOBL K256<>(SB), (NOPTR + RODATA), $512
  1032  

View as plain text