aboutsummaryrefslogtreecommitdiff
path: root/openssl/crypto/modes/asm/ghash-ia64.pl
blob: 0354c9544485e66d17728fac401ece51cd0c57fe (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
#!/usr/bin/env perl

# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# March 2010
#
# The module implements "4-bit" GCM GHASH function and underlying
# single multiplication operation in GF(2^128). "4-bit" means that it
# uses 256 bytes per-key table [+128 bytes shared table]. Streamed
# GHASH performance was measured to be 6.67 cycles per processed byte
# on Itanium 2, which is >90% better than Microsoft compiler generated
# code. To anchor to something else sha1-ia64.pl module processes one
# byte in 5.7 cycles. On Itanium GHASH should run at ~8.5 cycles per
# byte.

# September 2010
#
# It was originally thought that it makes lesser sense to implement
# "528B" variant on Itanium 2 for following reason. Because number of
# functional units is naturally limited, it appeared impossible to
# implement "528B" loop in 4 cycles, only in 5. This would mean that
# theoretically performance improvement couldn't be more than 20%.
# But occasionally you prove yourself wrong:-) I figured out a way to
# fold couple of instructions and having freed yet another instruction
# slot by unrolling the loop... Resulting performance is 4.45 cycles
# per processed byte and 50% better than "256B" version. On original
# Itanium performance should remain the same as the "256B" version,
# i.e. ~8.5 cycles.

$output=shift and (open STDOUT,">$output" or die "can't open $output: $!");

if ($^O eq "hpux") {
    $ADDP="addp4";
    for (@ARGV) { $ADDP="add" if (/[\+DD|\-mlp]64/); }
} else { $ADDP="add"; }
for (@ARGV)  {  $big_endian=1 if (/\-DB_ENDIAN/);
                $big_endian=0 if (/\-DL_ENDIAN/);  }
if (!defined($big_endian))
             {  $big_endian=(unpack('L',pack('N',1))==1);  }

sub loop() {
my $label=shift;
my ($p16,$p17)=(shift)?("p63","p63"):("p16","p17"); # mask references to inp

# Loop is scheduled for 6 ticks on Itanium 2 and 8 on Itanium, i.e.
# in scalable manner;-) Naturally assuming data in L1 cache...
# Special note about 'dep' instruction, which is used to construct
# &rem_4bit[Zlo&0xf]. It works, because rem_4bit is aligned at 128
# bytes boundary and lower 7 bits of its address are guaranteed to
# be zero.
$code.=<<___;
$label:
{ .mfi;	(p18)	ld8	Hlo=[Hi[1]],-8
	(p19)	dep	rem=Zlo,rem_4bitp,3,4	}
{ .mfi;	(p19)	xor	Zhi=Zhi,Hhi
	($p17)	xor	xi[1]=xi[1],in[1]	};;
{ .mfi;	(p18)	ld8	Hhi=[Hi[1]]
	(p19)	shrp	Zlo=Zhi,Zlo,4		}
{ .mfi;	(p19)	ld8	rem=[rem]
	(p18)	and	Hi[1]=mask0xf0,xi[2]	};;
{ .mmi;	($p16)	ld1	in[0]=[inp],-1
	(p18)	xor	Zlo=Zlo,Hlo
	(p19)	shr.u	Zhi=Zhi,4		}
{ .mib;	(p19)	xor	Hhi=Hhi,rem
	(p18)	add	Hi[1]=Htbl,Hi[1]	};;

{ .mfi;	(p18)	ld8	Hlo=[Hi[1]],-8
	(p18)	dep	rem=Zlo,rem_4bitp,3,4	}
{ .mfi;	(p17)	shladd	Hi[0]=xi[1],4,r0
	(p18)	xor	Zhi=Zhi,Hhi		};;
{ .mfi;	(p18)	ld8	Hhi=[Hi[1]]
	(p18)	shrp	Zlo=Zhi,Zlo,4		}
{ .mfi;	(p18)	ld8	rem=[rem]
	(p17)	and	Hi[0]=mask0xf0,Hi[0]	};;
{ .mmi;	(p16)	ld1	xi[0]=[Xi],-1
	(p18)	xor	Zlo=Zlo,Hlo
	(p18)	shr.u	Zhi=Zhi,4		}
{ .mib;	(p18)	xor	Hhi=Hhi,rem
	(p17)	add	Hi[0]=Htbl,Hi[0]
	br.ctop.sptk	$label			};;
___
}

$code=<<___;
.explicit
.text

prevfs=r2;	prevlc=r3;	prevpr=r8;
mask0xf0=r21;
rem=r22;	rem_4bitp=r23;
Xi=r24;		Htbl=r25;
inp=r26;	end=r27;
Hhi=r28;	Hlo=r29;
Zhi=r30;	Zlo=r31;

.align	128
.skip	16					// aligns loop body
.global	gcm_gmult_4bit#
.proc	gcm_gmult_4bit#
gcm_gmult_4bit:
	.prologue
{ .mmi;	.save	ar.pfs,prevfs
	alloc	prevfs=ar.pfs,2,6,0,8
	$ADDP	Xi=15,in0			// &Xi[15]
	mov	rem_4bitp=ip		}
{ .mii;	$ADDP	Htbl=8,in1			// &Htbl[0].lo
	.save	ar.lc,prevlc
	mov	prevlc=ar.lc
	.save	pr,prevpr
	mov	prevpr=pr		};;

	.body
	.rotr	in[3],xi[3],Hi[2]

{ .mib;	ld1	xi[2]=[Xi],-1			// Xi[15]
	mov	mask0xf0=0xf0
	brp.loop.imp	.Loop1,.Lend1-16};;
{ .mmi;	ld1	xi[1]=[Xi],-1			// Xi[14]
					};;
{ .mii;	shladd	Hi[1]=xi[2],4,r0
	mov	pr.rot=0x7<<16
	mov	ar.lc=13		};;
{ .mii;	and	Hi[1]=mask0xf0,Hi[1]
	mov	ar.ec=3
	xor	Zlo=Zlo,Zlo		};;
{ .mii;	add	Hi[1]=Htbl,Hi[1]		// &Htbl[nlo].lo
	add	rem_4bitp=rem_4bit#-gcm_gmult_4bit#,rem_4bitp
	xor	Zhi=Zhi,Zhi		};;
___
	&loop	(".Loop1",1);
$code.=<<___;
.Lend1:
{ .mib;	xor	Zhi=Zhi,Hhi		};;	// modulo-scheduling artefact
{ .mib;	mux1	Zlo=Zlo,\@rev		};;
{ .mib;	mux1	Zhi=Zhi,\@rev		};;
{ .mmi;	add	Hlo=9,Xi;;			// ;; is here to prevent
	add	Hhi=1,Xi		};;	// pipeline flush on Itanium
{ .mib;	st8	[Hlo]=Zlo
	mov	pr=prevpr,0x1ffff	};;
{ .mib;	st8	[Hhi]=Zhi
	mov	ar.lc=prevlc
	br.ret.sptk.many	b0	};;
.endp	gcm_gmult_4bit#
___

######################################################################
# "528B" (well, "512B" actualy) streamed GHASH
#
$Xip="in0";
$Htbl="in1";
$inp="in2";
$len="in3";
$rem_8bit="loc0";
$mask0xff="loc1";
($sum,$rum) = $big_endian ? ("nop.m","nop.m") : ("sum","rum");

sub load_htable() {
    for (my $i=0;$i<8;$i++) {
	$code.=<<___;
{ .mmi;	ld8	r`16+2*$i+1`=[r8],16		// Htable[$i].hi
	ld8	r`16+2*$i`=[r9],16	}	// Htable[$i].lo
{ .mmi;	ldf8	f`32+2*$i+1`=[r10],16		// Htable[`8+$i`].hi
	ldf8	f`32+2*$i`=[r11],16		// Htable[`8+$i`].lo
___
	$code.=shift	if (($i+$#_)==7);
	$code.="\t};;\n"
    }
}

$code.=<<___;
prevsp=r3;

.align	32
.skip	16					// aligns loop body
.global	gcm_ghash_4bit#
.proc	gcm_ghash_4bit#
gcm_ghash_4bit:
	.prologue
{ .mmi;	.save	ar.pfs,prevfs
	alloc	prevfs=ar.pfs,4,2,0,0
	.vframe	prevsp
	mov	prevsp=sp
	mov	$rem_8bit=ip		};;
	.body
{ .mfi;	$ADDP	r8=0+0,$Htbl
	$ADDP	r9=0+8,$Htbl		}
{ .mfi;	$ADDP	r10=128+0,$Htbl
	$ADDP	r11=128+8,$Htbl		};;
___
	&load_htable(
	"	$ADDP	$Xip=15,$Xip",		# &Xi[15]
	"	$ADDP	$len=$len,$inp",	# &inp[len]
	"	$ADDP	$inp=15,$inp",		# &inp[15]
	"	mov	$mask0xff=0xff",
	"	add	sp=-512,sp",
	"	andcm	sp=sp,$mask0xff",	# align stack frame
	"	add	r14=0,sp",
	"	add	r15=8,sp");
$code.=<<___;
{ .mmi;	$sum	1<<1				// go big-endian
	add	r8=256+0,sp
	add	r9=256+8,sp		}
{ .mmi;	add	r10=256+128+0,sp
	add	r11=256+128+8,sp
	add	$len=-17,$len		};;
___
for($i=0;$i<8;$i++) {	# generate first half of Hshr4[]
my ($rlo,$rhi)=("r".eval(16+2*$i),"r".eval(16+2*$i+1));
$code.=<<___;
{ .mmi;	st8	[r8]=$rlo,16			// Htable[$i].lo
	st8	[r9]=$rhi,16			// Htable[$i].hi
	shrp	$rlo=$rhi,$rlo,4	}//;;
{ .mmi;	stf8	[r10]=f`32+2*$i`,16		// Htable[`8+$i`].lo
	stf8	[r11]=f`32+2*$i+1`,16		// Htable[`8+$i`].hi
	shr.u	$rhi=$rhi,4		};;
{ .mmi;	st8	[r14]=$rlo,16			// Htable[$i].lo>>4
	st8	[r15]=$rhi,16		}//;;	// Htable[$i].hi>>4
___
}
$code.=<<___;
{ .mmi;	ld8	r16=[r8],16			// Htable[8].lo
	ld8	r17=[r9],16		};;	// Htable[8].hi
{ .mmi;	ld8	r18=[r8],16			// Htable[9].lo
	ld8	r19=[r9],16		}	// Htable[9].hi
{ .mmi;	rum	1<<5				// clear um.mfh
	shrp	r16=r17,r16,4		};;
___
for($i=0;$i<6;$i++) {	# generate second half of Hshr4[]
$code.=<<___;
{ .mmi;	ld8	r`20+2*$i`=[r8],16		// Htable[`10+$i`].lo
	ld8	r`20+2*$i+1`=[r9],16		// Htable[`10+$i`].hi
	shr.u	r`16+2*$i+1`=r`16+2*$i+1`,4	};;
{ .mmi;	st8	[r14]=r`16+2*$i`,16		// Htable[`8+$i`].lo>>4
	st8	[r15]=r`16+2*$i+1`,16		// Htable[`8+$i`].hi>>4
	shrp	r`18+2*$i`=r`18+2*$i+1`,r`18+2*$i`,4	}
___
}
$code.=<<___;
{ .mmi;	shr.u	r`16+2*$i+1`=r`16+2*$i+1`,4	};;
{ .mmi;	st8	[r14]=r`16+2*$i`,16		// Htable[`8+$i`].lo>>4
	st8	[r15]=r`16+2*$i+1`,16		// Htable[`8+$i`].hi>>4
	shrp	r`18+2*$i`=r`18+2*$i+1`,r`18+2*$i`,4	}
{ .mmi;	add	$Htbl=256,sp			// &Htable[0]
	add	$rem_8bit=rem_8bit#-gcm_ghash_4bit#,$rem_8bit
	shr.u	r`18+2*$i+1`=r`18+2*$i+1`,4	};;
{ .mmi;	st8	[r14]=r`18+2*$i`		// Htable[`8+$i`].lo>>4
	st8	[r15]=r`18+2*$i+1`	}	// Htable[`8+$i`].hi>>4
___

$in="r15";
@xi=("r16","r17");
@rem=("r18","r19");
($Alo,$Ahi,$Blo,$Bhi,$Zlo,$Zhi)=("r20","r21","r22","r23","r24","r25");
($Atbl,$Btbl)=("r26","r27");

$code.=<<___;	# (p16)
{ .mmi;	ld1	$in=[$inp],-1			//(p16) *inp--
	ld1	$xi[0]=[$Xip],-1		//(p16) *Xi--
	cmp.eq	p0,p6=r0,r0		};;	//	clear p6
___
push (@xi,shift(@xi)); push (@rem,shift(@rem));	# "rotate" registers

$code.=<<___;	# (p16),(p17)
{ .mmi;	ld1	$xi[0]=[$Xip],-1		//(p16) *Xi--
	xor	$xi[1]=$xi[1],$in	};;	//(p17) xi=$xi[i]^inp[i]
{ .mii;	ld1	$in=[$inp],-1			//(p16) *inp--
	dep	$Atbl=$xi[1],$Htbl,4,4		//(p17) &Htable[nlo].lo
	and	$xi[1]=-16,$xi[1]	};;	//(p17) nhi=xi&0xf0
.align	32
.LOOP:
{ .mmi;
(p6)	st8	[$Xip]=$Zhi,13
	xor	$Zlo=$Zlo,$Zlo
	add	$Btbl=$xi[1],$Htbl	};;	//(p17) &Htable[nhi].lo
___
push (@xi,shift(@xi)); push (@rem,shift(@rem));	# "rotate" registers

$code.=<<___;	# (p16),(p17),(p18)
{ .mmi;	ld8	$Alo=[$Atbl],8			//(p18) Htable[nlo].lo,&Htable[nlo].hi
	ld8	$rem[0]=[$Btbl],-256		//(p18) Htable[nhi].lo,&Hshr4[nhi].lo
	xor	$xi[1]=$xi[1],$in	};;	//(p17) xi=$xi[i]^inp[i]
{ .mfi;	ld8	$Ahi=[$Atbl]			//(p18) Htable[nlo].hi
	dep	$Atbl=$xi[1],$Htbl,4,4	}	//(p17) &Htable[nlo].lo
{ .mfi;	shladd	$rem[0]=$rem[0],4,r0		//(p18) Htable[nhi].lo<<4
	xor	$Zlo=$Zlo,$Alo		};;	//(p18) Z.lo^=Htable[nlo].lo
{ .mmi;	ld8	$Blo=[$Btbl],8			//(p18) Hshr4[nhi].lo,&Hshr4[nhi].hi
	ld1	$in=[$inp],-1		}	//(p16) *inp--
{ .mmi;	xor	$rem[0]=$rem[0],$Zlo		//(p18) Z.lo^(Htable[nhi].lo<<4)
	mov	$Zhi=$Ahi			//(p18) Z.hi^=Htable[nlo].hi
	and	$xi[1]=-16,$xi[1]	};;	//(p17) nhi=xi&0xf0
{ .mmi;	ld8	$Bhi=[$Btbl]			//(p18) Hshr4[nhi].hi
	ld1	$xi[0]=[$Xip],-1		//(p16) *Xi--
	shrp	$Zlo=$Zhi,$Zlo,8	}	//(p18) Z.lo=(Z.hi<<56)|(Z.lo>>8)
{ .mmi;	and	$rem[0]=$rem[0],$mask0xff	//(p18) rem=($Zlo^(Htable[nhi].lo<<4))&0xff
	add	$Btbl=$xi[1],$Htbl	};;	//(p17) &Htable[nhi]
___
push (@xi,shift(@xi)); push (@rem,shift(@rem));	# "rotate" registers

for ($i=1;$i<14;$i++) {
# Above and below fragments are derived from this one by removing
# unsuitable (p??) instructions.
$code.=<<___;	# (p16),(p17),(p18),(p19)
{ .mmi;	ld8	$Alo=[$Atbl],8			//(p18) Htable[nlo].lo,&Htable[nlo].hi
	ld8	$rem[0]=[$Btbl],-256		//(p18) Htable[nhi].lo,&Hshr4[nhi].lo
	shr.u	$Zhi=$Zhi,8		}	//(p19) Z.hi>>=8
{ .mmi;	shladd	$rem[1]=$rem[1],1,$rem_8bit	//(p19) &rem_8bit[rem]
	xor	$Zlo=$Zlo,$Blo			//(p19) Z.lo^=Hshr4[nhi].lo
	xor	$xi[1]=$xi[1],$in	};;	//(p17) xi=$xi[i]^inp[i]
{ .mmi;	ld8	$Ahi=[$Atbl]			//(p18) Htable[nlo].hi
	ld2	$rem[1]=[$rem[1]]		//(p19) rem_8bit[rem]
	dep	$Atbl=$xi[1],$Htbl,4,4	}	//(p17) &Htable[nlo].lo
{ .mmi;	shladd	$rem[0]=$rem[0],4,r0		//(p18) Htable[nhi].lo<<4
	xor	$Zlo=$Zlo,$Alo			//(p18) Z.lo^=Htable[nlo].lo
	xor	$Zhi=$Zhi,$Bhi		};;	//(p19) Z.hi^=Hshr4[nhi].hi
{ .mmi;	ld8	$Blo=[$Btbl],8			//(p18) Hshr4[nhi].lo,&Hshr4[nhi].hi
	ld1	$in=[$inp],-1			//(p16) *inp--
	shl	$rem[1]=$rem[1],48	}	//(p19) rem_8bit[rem]<<48
{ .mmi;	xor	$rem[0]=$rem[0],$Zlo		//(p18) Z.lo^(Htable[nhi].lo<<4)
	xor	$Zhi=$Zhi,$Ahi			//(p18) Z.hi^=Htable[nlo].hi
	and	$xi[1]=-16,$xi[1]	};;	//(p17) nhi=xi&0xf0
{ .mmi;	ld8	$Bhi=[$Btbl]			//(p18) Hshr4[nhi].hi
	ld1	$xi[0]=[$Xip],-1		//(p16) *Xi--
	shrp	$Zlo=$Zhi,$Zlo,8	}	//(p18) Z.lo=(Z.hi<<56)|(Z.lo>>8)
{ .mmi;	and	$rem[0]=$rem[0],$mask0xff	//(p18) rem=($Zlo^(Htable[nhi].lo<<4))&0xff
	xor	$Zhi=$Zhi,$rem[1]		//(p19) Z.hi^=rem_8bit[rem]<<48
	add	$Btbl=$xi[1],$Htbl	};;	//(p17) &Htable[nhi]
___
push (@xi,shift(@xi)); push (@rem,shift(@rem));	# "rotate" registers
}

$code.=<<___;	# (p17),(p18),(p19)
{ .mmi;	ld8	$Alo=[$Atbl],8			//(p18) Htable[nlo].lo,&Htable[nlo].hi
	ld8	$rem[0]=[$Btbl],-256		//(p18) Htable[nhi].lo,&Hshr4[nhi].lo
	shr.u	$Zhi=$Zhi,8		}	//(p19) Z.hi>>=8
{ .mmi;	shladd	$rem[1]=$rem[1],1,$rem_8bit	//(p19) &rem_8bit[rem]
	xor	$Zlo=$Zlo,$Blo			//(p19) Z.lo^=Hshr4[nhi].lo
	xor	$xi[1]=$xi[1],$in	};;	//(p17) xi=$xi[i]^inp[i]
{ .mmi;	ld8	$Ahi=[$Atbl]			//(p18) Htable[nlo].hi
	ld2	$rem[1]=[$rem[1]]		//(p19) rem_8bit[rem]
	dep	$Atbl=$xi[1],$Htbl,4,4	};;	//(p17) &Htable[nlo].lo
{ .mmi;	shladd	$rem[0]=$rem[0],4,r0		//(p18) Htable[nhi].lo<<4
	xor	$Zlo=$Zlo,$Alo			//(p18) Z.lo^=Htable[nlo].lo
	xor	$Zhi=$Zhi,$Bhi		};;	//(p19) Z.hi^=Hshr4[nhi].hi
{ .mmi;	ld8	$Blo=[$Btbl],8			//(p18) Hshr4[nhi].lo,&Hshr4[nhi].hi
	shl	$rem[1]=$rem[1],48	}	//(p19) rem_8bit[rem]<<48
{ .mmi;	xor	$rem[0]=$rem[0],$Zlo		//(p18) Z.lo^(Htable[nhi].lo<<4)
	xor	$Zhi=$Zhi,$Ahi			//(p18) Z.hi^=Htable[nlo].hi
	and	$xi[1]=-16,$xi[1]	};;	//(p17) nhi=xi&0xf0
{ .mmi;	ld8	$Bhi=[$Btbl]			//(p18) Hshr4[nhi].hi
	shrp	$Zlo=$Zhi,$Zlo,8	}	//(p18) Z.lo=(Z.hi<<56)|(Z.lo>>8)
{ .mmi;	and	$rem[0]=$rem[0],$mask0xff	//(p18) rem=($Zlo^(Htable[nhi].lo<<4))&0xff
	xor	$Zhi=$Zhi,$rem[1]		//(p19) Z.hi^=rem_8bit[rem]<<48
	add	$Btbl=$xi[1],$Htbl	};;	//(p17) &Htable[nhi]
___
push (@xi,shift(@xi)); push (@rem,shift(@rem));	# "rotate" registers

$code.=<<___;	# (p18),(p19)
{ .mfi;	ld8	$Alo=[$Atbl],8			//(p18) Htable[nlo].lo,&Htable[nlo].hi
	shr.u	$Zhi=$Zhi,8		}	//(p19) Z.hi>>=8
{ .mfi;	shladd	$rem[1]=$rem[1],1,$rem_8bit	//(p19) &rem_8bit[rem]
	xor	$Zlo=$Zlo,$Blo		};;	//(p19) Z.lo^=Hshr4[nhi].lo
{ .mfi;	ld8	$Ahi=[$Atbl]			//(p18) Htable[nlo].hi
	xor	$Zlo=$Zlo,$Alo		}	//(p18) Z.lo^=Htable[nlo].lo
{ .mfi;	ld2	$rem[1]=[$rem[1]]		//(p19) rem_8bit[rem]
	xor	$Zhi=$Zhi,$Bhi		};;	//(p19) Z.hi^=Hshr4[nhi].hi
{ .mfi;	ld8	$Blo=[$Btbl],8			//(p18) Htable[nhi].lo,&Htable[nhi].hi
	shl	$rem[1]=$rem[1],48	}	//(p19) rem_8bit[rem]<<48
{ .mfi;	shladd	$rem[0]=$Zlo,4,r0		//(p18) Z.lo<<4
	xor	$Zhi=$Zhi,$Ahi		};;	//(p18) Z.hi^=Htable[nlo].hi
{ .mfi;	ld8	$Bhi=[$Btbl]			//(p18) Htable[nhi].hi
	shrp	$Zlo=$Zhi,$Zlo,4	}	//(p18) Z.lo=(Z.hi<<60)|(Z.lo>>4)
{ .mfi;	and	$rem[0]=$rem[0],$mask0xff	//(p18) rem=($Zlo^(Htable[nhi].lo<<4))&0xff
	xor	$Zhi=$Zhi,$rem[1]	};;	//(p19) Z.hi^=rem_8bit[rem]<<48
___
push (@xi,shift(@xi)); push (@rem,shift(@rem));	# "rotate" registers

$code.=<<___;	# (p19)
{ .mmi;	cmp.ltu	p6,p0=$inp,$len
	add	$inp=32,$inp
	shr.u	$Zhi=$Zhi,4		}	//(p19) Z.hi>>=4
{ .mmi;	shladd	$rem[1]=$rem[1],1,$rem_8bit	//(p19) &rem_8bit[rem]
	xor	$Zlo=$Zlo,$Blo			//(p19) Z.lo^=Hshr4[nhi].lo
	add	$Xip=9,$Xip		};;	//	&Xi.lo
{ .mmi;	ld2	$rem[1]=[$rem[1]]		//(p19) rem_8bit[rem]
(p6)	ld1	$in=[$inp],-1			//[p16] *inp--
(p6)	extr.u	$xi[1]=$Zlo,8,8		}	//[p17] Xi[14]
{ .mmi;	xor	$Zhi=$Zhi,$Bhi			//(p19) Z.hi^=Hshr4[nhi].hi
(p6)	and	$xi[0]=$Zlo,$mask0xff	};;	//[p16] Xi[15]
{ .mmi;	st8	[$Xip]=$Zlo,-8
(p6)	xor	$xi[0]=$xi[0],$in		//[p17] xi=$xi[i]^inp[i]
	shl	$rem[1]=$rem[1],48	};;	//(p19) rem_8bit[rem]<<48
{ .mmi;
(p6)	ld1	$in=[$inp],-1			//[p16] *inp--
	xor	$Zhi=$Zhi,$rem[1]		//(p19) Z.hi^=rem_8bit[rem]<<48
(p6)	dep	$Atbl=$xi[0],$Htbl,4,4	}	//[p17] &Htable[nlo].lo
{ .mib;
(p6)	and	$xi[0]=-16,$xi[0]		//[p17] nhi=xi&0xf0
(p6)	br.cond.dptk.many	.LOOP	};;

{ .mib;	st8	[$Xip]=$Zhi		};;
{ .mib;	$rum	1<<1				// return to little-endian
	.restore	sp
	mov	sp=prevsp
	br.ret.sptk.many	b0	};;
.endp	gcm_ghash_4bit#
___
$code.=<<___;
.align	128
.type	rem_4bit#,\@object
rem_4bit:
        data8	0x0000<<48, 0x1C20<<48, 0x3840<<48, 0x2460<<48
        data8	0x7080<<48, 0x6CA0<<48, 0x48C0<<48, 0x54E0<<48
        data8	0xE100<<48, 0xFD20<<48, 0xD940<<48, 0xC560<<48
        data8	0x9180<<48, 0x8DA0<<48, 0xA9C0<<48, 0xB5E0<<48
.size	rem_4bit#,128
.type	rem_8bit#,\@object
rem_8bit:
	data1	0x00,0x00, 0x01,0xC2, 0x03,0x84, 0x02,0x46, 0x07,0x08, 0x06,0xCA, 0x04,0x8C, 0x05,0x4E
	data1	0x0E,0x10, 0x0F,0xD2, 0x0D,0x94, 0x0C,0x56, 0x09,0x18, 0x08,0xDA, 0x0A,0x9C, 0x0B,0x5E
	data1	0x1C,0x20, 0x1D,0xE2, 0x1F,0xA4, 0x1E,0x66, 0x1B,0x28, 0x1A,0xEA, 0x18,0xAC, 0x19,0x6E
	data1	0x12,0x30, 0x13,0xF2, 0x11,0xB4, 0x10,0x76, 0x15,0x38, 0x14,0xFA, 0x16,0xBC, 0x17,0x7E
	data1	0x38,0x40, 0x39,0x82, 0x3B,0xC4, 0x3A,0x06, 0x3F,0x48, 0x3E,0x8A, 0x3C,0xCC, 0x3D,0x0E
	data1	0x36,0x50, 0x37,0x92, 0x35,0xD4, 0x34,0x16, 0x31,0x58, 0x30,0x9A, 0x32,0xDC, 0x33,0x1E
	data1	0x24,0x60, 0x25,0xA2, 0x27,0xE4, 0x26,0x26, 0x23,0x68, 0x22,0xAA, 0x20,0xEC, 0x21,0x2E
	data1	0x2A,0x70, 0x2B,0xB2, 0x29,0xF4, 0x28,0x36, 0x2D,0x78, 0x2C,0xBA, 0x2E,0xFC, 0x2F,0x3E
	data1	0x70,0x80, 0x71,0x42, 0x73,0x04, 0x72,0xC6, 0x77,0x88, 0x76,0x4A, 0x74,0x0C, 0x75,0xCE
	data1	0x7E,0x90, 0x7F,0x52, 0x7D,0x14, 0x7C,0xD6, 0x79,0x98, 0x78,0x5A, 0x7A,0x1C, 0x7B,0xDE
	data1	0x6C,0xA0, 0x6D,0x62, 0x6F,0x24, 0x6E,0xE6, 0x6B,0xA8, 0x6A,0x6A, 0x68,0x2C, 0x69,0xEE
	data1	0x62,0xB0, 0x63,0x72, 0x61,0x34, 0x60,0xF6, 0x65,0xB8, 0x64,0x7A, 0x66,0x3C, 0x67,0xFE
	data1	0x48,0xC0, 0x49,0x02, 0x4B,0x44, 0x4A,0x86, 0x4F,0xC8, 0x4E,0x0A, 0x4C,0x4C, 0x4D,0x8E
	data1	0x46,0xD0, 0x47,0x12, 0x45,0x54, 0x44,0x96, 0x41,0xD8, 0x40,0x1A, 0x42,0x5C, 0x43,0x9E
	data1	0x54,0xE0, 0x55,0x22, 0x57,0x64, 0x56,0xA6, 0x53,0xE8, 0x52,0x2A, 0x50,0x6C, 0x51,0xAE
	data1	0x5A,0xF0, 0x5B,0x32, 0x59,0x74, 0x58,0xB6, 0x5D,0xF8, 0x5C,0x3A, 0x5E,0x7C, 0x5F,0xBE
	data1	0xE1,0x00, 0xE0,0xC2, 0xE2,0x84, 0xE3,0x46, 0xE6,0x08, 0xE7,0xCA, 0xE5,0x8C, 0xE4,0x4E
	data1	0xEF,0x10, 0xEE,0xD2, 0xEC,0x94, 0xED,0x56, 0xE8,0x18, 0xE9,0xDA, 0xEB,0x9C, 0xEA,0x5E
	data1	0xFD,0x20, 0xFC,0xE2, 0xFE,0xA4, 0xFF,0x66, 0xFA,0x28, 0xFB,0xEA, 0xF9,0xAC, 0xF8,0x6E
	data1	0xF3,0x30, 0xF2,0xF2, 0xF0,0xB4, 0xF1,0x76, 0xF4,0x38, 0xF5,0xFA, 0xF7,0xBC, 0xF6,0x7E
	data1	0xD9,0x40, 0xD8,0x82, 0xDA,0xC4, 0xDB,0x06, 0xDE,0x48, 0xDF,0x8A, 0xDD,0xCC, 0xDC,0x0E
	data1	0xD7,0x50, 0xD6,0x92, 0xD4,0xD4, 0xD5,0x16, 0xD0,0x58, 0xD1,0x9A, 0xD3,0xDC, 0xD2,0x1E
	data1	0xC5,0x60, 0xC4,0xA2, 0xC6,0xE4, 0xC7,0x26, 0xC2,0x68, 0xC3,0xAA, 0xC1,0xEC, 0xC0,0x2E
	data1	0xCB,0x70, 0xCA,0xB2, 0xC8,0xF4, 0xC9,0x36, 0xCC,0x78, 0xCD,0xBA, 0xCF,0xFC, 0xCE,0x3E
	data1	0x91,0x80, 0x90,0x42, 0x92,0x04, 0x93,0xC6, 0x96,0x88, 0x97,0x4A, 0x95,0x0C, 0x94,0xCE
	data1	0x9F,0x90, 0x9E,0x52, 0x9C,0x14, 0x9D,0xD6, 0x98,0x98, 0x99,0x5A, 0x9B,0x1C, 0x9A,0xDE
	data1	0x8D,0xA0, 0x8C,0x62, 0x8E,0x24, 0x8F,0xE6, 0x8A,0xA8, 0x8B,0x6A, 0x89,0x2C, 0x88,0xEE
	data1	0x83,0xB0, 0x82,0x72, 0x80,0x34, 0x81,0xF6, 0x84,0xB8, 0x85,0x7A, 0x87,0x3C, 0x86,0xFE
	data1	0xA9,0xC0, 0xA8,0x02, 0xAA,0x44, 0xAB,0x86, 0xAE,0xC8, 0xAF,0x0A, 0xAD,0x4C, 0xAC,0x8E
	data1	0xA7,0xD0, 0xA6,0x12, 0xA4,0x54, 0xA5,0x96, 0xA0,0xD8, 0xA1,0x1A, 0xA3,0x5C, 0xA2,0x9E
	data1	0xB5,0xE0, 0xB4,0x22, 0xB6,0x64, 0xB7,0xA6, 0xB2,0xE8, 0xB3,0x2A, 0xB1,0x6C, 0xB0,0xAE
	data1	0xBB,0xF0, 0xBA,0x32, 0xB8,0x74, 0xB9,0xB6, 0xBC,0xF8, 0xBD,0x3A, 0xBF,0x7C, 0xBE,0xBE
.size	rem_8bit#,512
stringz	"GHASH for IA64, CRYPTOGAMS by <appro\@openssl.org>"
___

$code =~ s/mux1(\s+)\S+\@rev/nop.i$1 0x0/gm      if ($big_endian);
$code =~ s/\`([^\`]*)\`/eval $1/gem;

print $code;
close STDOUT;