ChangeSet 1.2181.47.2, 2005/03/29 13:46:30-08:00, [EMAIL PROTECTED]
[SPARC64]: More g5 register usage elimination.
Use temporal stores and prefetches in memset/bzero
while we are at it.
Signed-off-by: David S. Miller <[EMAIL PROTECTED]>
arch/sparc64/lib/VISbzero.S | 274 --------------------------------------
arch/sparc64/lib/VISmemset.S | 240 ---------------------------------
b/arch/sparc64/lib/Makefile | 2
b/arch/sparc64/lib/U1memcpy.S | 36 ++--
b/arch/sparc64/lib/U3memcpy.S | 28 ++-
b/arch/sparc64/lib/atomic.S | 64 ++++----
b/arch/sparc64/lib/bzero.S | 158 +++++++++++++++++++++
b/arch/sparc64/lib/debuglocks.c | 76 +++++-----
b/arch/sparc64/lib/dec_and_lock.S | 16 +-
b/arch/sparc64/lib/mcount.S | 18 +-
b/arch/sparc64/lib/memcmp.S | 4
b/arch/sparc64/lib/memmove.S | 10 -
b/arch/sparc64/lib/memscan.S | 32 ++--
b/arch/sparc64/lib/strlen.S | 12 -
b/arch/sparc64/lib/strlen_user.S | 12 -
b/arch/sparc64/lib/xor.S | 46 +++---
b/arch/sparc64/mm/init.c | 24 +--
b/include/asm-sparc64/system.h | 36 ++--
18 files changed, 370 insertions(+), 718 deletions(-)
diff -Nru a/arch/sparc64/lib/Makefile b/arch/sparc64/lib/Makefile
--- a/arch/sparc64/lib/Makefile 2005-04-03 21:17:36 -07:00
+++ b/arch/sparc64/lib/Makefile 2005-04-03 21:17:36 -07:00
@@ -7,7 +7,7 @@
lib-y := PeeCeeI.o copy_page.o clear_page.o strlen.o strncmp.o \
memscan.o strncpy_from_user.o strlen_user.o memcmp.o checksum.o \
- VISbzero.o VISmemset.o VIScsum.o VIScsumcopy.o \
+ bzero.o VIScsum.o VIScsumcopy.o \
VIScsumcopyusr.o VISsave.o atomic.o bitops.o \
U1memcpy.o U1copy_from_user.o U1copy_to_user.o \
U3memcpy.o U3copy_from_user.o U3copy_to_user.o U3patch.o \
diff -Nru a/arch/sparc64/lib/U1memcpy.S b/arch/sparc64/lib/U1memcpy.S
--- a/arch/sparc64/lib/U1memcpy.S 2005-04-03 21:17:36 -07:00
+++ b/arch/sparc64/lib/U1memcpy.S 2005-04-03 21:17:36 -07:00
@@ -7,6 +7,7 @@
#ifdef __KERNEL__
#include <asm/visasm.h>
#include <asm/asi.h>
+#define GLOBAL_SPARE %g7
#else
#define ASI_BLK_P 0xf0
#define FPRS_FEF 0x04
@@ -18,6 +19,7 @@
#define VISEntry rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs
#define VISExit and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
#endif
+#define GLOBAL_SPARE %g5
#endif
#ifndef EX_LD
@@ -123,7 +125,7 @@
cmp %g2, 0
tne %xcc, 5
PREAMBLE
- mov %o0, %g5
+ mov %o0, %o4
cmp %o2, 0
be,pn %XCC, 85f
or %o0, %o1, %o3
@@ -146,7 +148,7 @@
* of bytes to copy to make 'dst' 64-byte aligned. We pre-
* subtract this from 'len'.
*/
- sub %o0, %o1, %o4
+ sub %o0, %o1, GLOBAL_SPARE
sub %g2, 0x40, %g2
sub %g0, %g2, %g2
sub %o2, %g2, %o2
@@ -156,11 +158,11 @@
1: subcc %g1, 0x1, %g1
EX_LD(LOAD(ldub, %o1 + 0x00, %o3))
- EX_ST(STORE(stb, %o3, %o1 + %o4))
+ EX_ST(STORE(stb, %o3, %o1 + GLOBAL_SPARE))
bgu,pt %XCC, 1b
add %o1, 0x1, %o1
- add %o1, %o4, %o0
+ add %o1, GLOBAL_SPARE, %o0
2: cmp %g2, 0x0
and %o1, 0x7, %g1
@@ -188,19 +190,19 @@
3:
membar #LoadStore | #StoreStore | #StoreLoad
- subcc %o2, 0x40, %o4
+ subcc %o2, 0x40, GLOBAL_SPARE
add %o1, %g1, %g1
- andncc %o4, (0x40 - 1), %o4
+ andncc GLOBAL_SPARE, (0x40 - 1), GLOBAL_SPARE
srl %g1, 3, %g2
- sub %o2, %o4, %g3
+ sub %o2, GLOBAL_SPARE, %g3
andn %o1, (0x40 - 1), %o1
and %g2, 7, %g2
andncc %g3, 0x7, %g3
fmovd %f0, %f2
sub %g3, 0x8, %g3
- sub %o2, %o4, %o2
+ sub %o2, GLOBAL_SPARE, %o2
- add %g1, %o4, %g1
+ add %g1, GLOBAL_SPARE, %g1
subcc %o2, %g3, %o2
EX_LD(LOAD_BLK(%o1, %f0))
@@ -208,7 +210,7 @@
add %g1, %g3, %g1
EX_LD(LOAD_BLK(%o1, %f16))
add %o1, 0x40, %o1
- sub %o4, 0x80, %o4
+ sub GLOBAL_SPARE, 0x80, GLOBAL_SPARE
EX_LD(LOAD_BLK(%o1, %f32))
add %o1, 0x40, %o1
@@ -449,18 +451,18 @@
2: membar #StoreLoad | #StoreStore
VISExit
retl
- mov EX_RETVAL(%g5), %o0
+ mov EX_RETVAL(%o4), %o0
.align 64
70: /* 16 < len <= (5 * 64) */
bne,pn %XCC, 75f
sub %o0, %o1, %o3
-72: andn %o2, 0xf, %o4
+72: andn %o2, 0xf, GLOBAL_SPARE
and %o2, 0xf, %o2
1: EX_LD(LOAD(ldx, %o1 + 0x00, %o5))
EX_LD(LOAD(ldx, %o1 + 0x08, %g1))
- subcc %o4, 0x10, %o4
+ subcc GLOBAL_SPARE, 0x10, GLOBAL_SPARE
EX_ST(STORE(stx, %o5, %o1 + %o3))
add %o1, 0x8, %o1
EX_ST(STORE(stx, %g1, %o1 + %o3))
@@ -512,10 +514,10 @@
andn %o1, 0x7, %o1
EX_LD(LOAD(ldx, %o1, %g2))
sub %o3, %g1, %o3
- andn %o2, 0x7, %o4
+ andn %o2, 0x7, GLOBAL_SPARE
sllx %g2, %g1, %g2
1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3))
- subcc %o4, 0x8, %o4
+ subcc GLOBAL_SPARE, 0x8, GLOBAL_SPARE
add %o1, 0x8, %o1
srlx %g3, %o3, %o5
or %o5, %g2, %o5
@@ -544,7 +546,7 @@
add %o1, 4, %o1
85: retl
- mov EX_RETVAL(%g5), %o0
+ mov EX_RETVAL(%o4), %o0
.align 32
90: EX_LD(LOAD(ldub, %o1, %g1))
@@ -553,6 +555,6 @@
bgu,pt %XCC, 90b
add %o1, 1, %o1
retl
- mov EX_RETVAL(%g5), %o0
+ mov EX_RETVAL(%o4), %o0
.size FUNC_NAME, .-FUNC_NAME
diff -Nru a/arch/sparc64/lib/U3memcpy.S b/arch/sparc64/lib/U3memcpy.S
--- a/arch/sparc64/lib/U3memcpy.S 2005-04-03 21:17:36 -07:00
+++ b/arch/sparc64/lib/U3memcpy.S 2005-04-03 21:17:36 -07:00
@@ -6,6 +6,7 @@
#ifdef __KERNEL__
#include <asm/visasm.h>
#include <asm/asi.h>
+#define GLOBAL_SPARE %g7
#else
#define ASI_BLK_P 0xf0
#define FPRS_FEF 0x04
@@ -17,6 +18,7 @@
#define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs
#define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
#endif
+#define GLOBAL_SPARE %g5
#endif
#ifndef EX_LD
@@ -84,7 +86,7 @@
cmp %g2, 0
tne %xcc, 5
PREAMBLE
- mov %o0, %g5
+ mov %o0, %o4
cmp %o2, 0
be,pn %XCC, 85f
or %o0, %o1, %o3
@@ -109,7 +111,7 @@
* of bytes to copy to make 'dst' 64-byte aligned. We pre-
* subtract this from 'len'.
*/
- sub %o0, %o1, %o4
+ sub %o0, %o1, GLOBAL_SPARE
sub %g2, 0x40, %g2
sub %g0, %g2, %g2
sub %o2, %g2, %o2
@@ -119,11 +121,11 @@
1: subcc %g1, 0x1, %g1
EX_LD(LOAD(ldub, %o1 + 0x00, %o3))
- EX_ST(STORE(stb, %o3, %o1 + %o4))
+ EX_ST(STORE(stb, %o3, %o1 + GLOBAL_SPARE))
bgu,pt %XCC, 1b
add %o1, 0x1, %o1
- add %o1, %o4, %o0
+ add %o1, GLOBAL_SPARE, %o0
2: cmp %g2, 0x0
and %o1, 0x7, %g1
@@ -149,7 +151,7 @@
3: LOAD(prefetch, %o1 + 0x000, #one_read)
LOAD(prefetch, %o1 + 0x040, #one_read)
- andn %o2, (0x40 - 1), %o4
+ andn %o2, (0x40 - 1), GLOBAL_SPARE
LOAD(prefetch, %o1 + 0x080, #one_read)
LOAD(prefetch, %o1 + 0x0c0, #one_read)
LOAD(prefetch, %o1 + 0x100, #one_read)
@@ -173,10 +175,10 @@
faligndata %f10, %f12, %f26
EX_LD(LOAD(ldd, %o1 + 0x040, %f0))
- subcc %o4, 0x80, %o4
+ subcc GLOBAL_SPARE, 0x80, GLOBAL_SPARE
add %o1, 0x40, %o1
bgu,pt %XCC, 1f
- srl %o4, 6, %o3
+ srl GLOBAL_SPARE, 6, %o3
ba,pt %xcc, 2f
nop
@@ -315,9 +317,9 @@
sub %o0, %o1, %o3
72:
- andn %o2, 0xf, %o4
+ andn %o2, 0xf, GLOBAL_SPARE
and %o2, 0xf, %o2
-1: subcc %o4, 0x10, %o4
+1: subcc GLOBAL_SPARE, 0x10, GLOBAL_SPARE
EX_LD(LOAD(ldx, %o1 + 0x00, %o5))
EX_LD(LOAD(ldx, %o1 + 0x08, %g1))
EX_ST(STORE(stx, %o5, %o1 + %o3))
@@ -372,10 +374,10 @@
andn %o1, 0x7, %o1
EX_LD(LOAD(ldx, %o1, %g2))
sub %o3, %g1, %o3
- andn %o2, 0x7, %o4
+ andn %o2, 0x7, GLOBAL_SPARE
sllx %g2, %g1, %g2
1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3))
- subcc %o4, 0x8, %o4
+ subcc GLOBAL_SPARE, 0x8, GLOBAL_SPARE
add %o1, 0x8, %o1
srlx %g3, %o3, %o5
or %o5, %g2, %o5
@@ -405,7 +407,7 @@
add %o1, 4, %o1
85: retl
- mov EX_RETVAL(%g5), %o0
+ mov EX_RETVAL(%o4), %o0
.align 32
90:
@@ -415,6 +417,6 @@
bgu,pt %XCC, 90b
add %o1, 1, %o1
retl
- mov EX_RETVAL(%g5), %o0
+ mov EX_RETVAL(%o4), %o0
.size FUNC_NAME, .-FUNC_NAME
diff -Nru a/arch/sparc64/lib/VISbzero.S b/arch/sparc64/lib/VISbzero.S
--- a/arch/sparc64/lib/VISbzero.S 2005-04-03 21:17:36 -07:00
+++ /dev/null Wed Dec 31 16:00:00 196900
@@ -1,274 +0,0 @@
-/* $Id: VISbzero.S,v 1.11 2001/03/15 08:51:24 anton Exp $
- * VISbzero.S: High speed clear operations utilizing the UltraSparc
- * Visual Instruction Set.
- *
- * Copyright (C) 1997 David S. Miller ([EMAIL PROTECTED])
- * Copyright (C) 1996, 1997, 1999 Jakub Jelinek ([EMAIL PROTECTED])
- */
-
-#include "VIS.h"
-
-#ifdef __KERNEL__
-#include <asm/visasm.h>
-
-#define EXN(x,y,a,b,z) \
-98: x,y; \
- .section .fixup; \
- .align 4; \
-99: ba VISbzerofixup_ret##z; \
- a, b, %o0; \
- .section __ex_table; \
- .align 4; \
- .word 98b, 99b; \
- .text; \
- .align 4;
-#define EXC(x,y,a,b,c...) \
-98: x,y; \
- .section .fixup; \
- .align 4; \
-99: c; \
- ba VISbzerofixup_ret0; \
- a, b, %o0; \
- .section __ex_table; \
- .align 4; \
- .word 98b, 99b; \
- .text; \
- .align 4;
-#define EXO1(x,y) \
-98: x,y; \
- .section __ex_table; \
- .align 4; \
- .word 98b, VISbzerofixup_reto1; \
- .text; \
- .align 4;
-#define EX(x,y,a,b) EXN(x,y,a,b,0)
-#define EX1(x,y,a,b) EXN(x,y,a,b,1)
-#define EX2(x,y,a,b) EXN(x,y,a,b,2)
-#define EXT(start,end,handler) \
- .section __ex_table; \
- .align 4; \
- .word start, 0, end, handler; \
- .text; \
- .align 4
-#else
-#define EX(x,y,a,b) x,y
-#define EX1(x,y,a,b) x,y
-#define EX2(x,y,a,b) x,y
-#define EXC(x,y,a,b,c...) x,y
-#define EXO1(x,y) x,y
-#define EXT(a,b,c)
-#endif
-
-#define ZERO_BLOCKS(base, offset, source) \
- STX source, [base - offset - 0x38] ASINORMAL; \
- STX source, [base - offset - 0x30] ASINORMAL; \
- STX source, [base - offset - 0x28] ASINORMAL; \
- STX source, [base - offset - 0x20] ASINORMAL; \
- STX source, [base - offset - 0x18] ASINORMAL; \
- STX source, [base - offset - 0x10] ASINORMAL; \
- STX source, [base - offset - 0x08] ASINORMAL; \
- STX source, [base - offset - 0x00] ASINORMAL;
-
-#ifdef __KERNEL__
-#define RETL clr %o0
-#else
-#define RETL mov %g3, %o0
-#endif
-
- /* Well, bzero is a lot easier to get right than bcopy... */
-#ifdef __KERNEL__
- .section __ex_table,#alloc
- .section .fixup,#alloc,#execinstr
-#endif
- .text
- .align 32
-#ifdef __KERNEL__
- .globl __bzero, __bzero_noasi
-__bzero_noasi:
- rd %asi, %g5
- ba,pt %xcc, __bzero+12
- mov %g5, %o4
-__bzero:
- rd %asi, %g5
- wr %g0, ASI_P, %asi ! LSU Group
- mov ASI_P, %o4
-#else
- .globl bzero
-bzero_private:
-bzero:
-#ifndef REGS_64BIT
- srl %o1, 0, %o1
-#endif
- mov %o0, %g3
-#endif
- cmp %o1, 7
- bleu,pn %xcc, 17f
- andcc %o0, 3, %o2
- be,a,pt %xcc, 4f
- andcc %o0, 4, %g0
- cmp %o2, 3
- be,pn %xcc, 2f
- EXO1(STB %g0, [%o0 + 0x00] ASINORMAL)
- cmp %o2, 2
- be,pt %xcc, 2f
- EX(STB %g0, [%o0 + 0x01] ASINORMAL, sub %o1, 1)
- EX(STB %g0, [%o0 + 0x02] ASINORMAL, sub %o1, 2)
-2: sub %o2, 4, %o2
- sub %o0, %o2, %o0
- add %o1, %o2, %o1
- andcc %o0, 4, %g0
-4: be,pt %xcc, 2f
- cmp %o1, 128
- EXO1(STW %g0, [%o0] ASINORMAL)
- sub %o1, 4, %o1
- add %o0, 4, %o0
-2: blu,pn %xcc, 9f
- andcc %o0, 0x38, %o2
- be,pn %icc, 6f
- mov 64, %o5
- andcc %o0, 8, %g0
- be,pn %icc, 1f
- sub %o5, %o2, %o5
- EX(STX %g0, [%o0] ASINORMAL, sub %o1, 0)
- add %o0, 8, %o0
-1: andcc %o5, 16, %g0
- be,pn %icc, 1f
- sub %o1, %o5, %o1
- EX1(STX %g0, [%o0] ASINORMAL, add %g0, 0)
- EX1(STX %g0, [%o0 + 8] ASINORMAL, sub %g0, 8)
- add %o0, 16, %o0
-1: andcc %o5, 32, %g0
- be,pn %icc, 7f
- andncc %o1, 0x3f, %o3
- EX(STX %g0, [%o0] ASINORMAL, add %o1, 32)
- EX(STX %g0, [%o0 + 8] ASINORMAL, add %o1, 24)
- EX(STX %g0, [%o0 + 16] ASINORMAL, add %o1, 16)
- EX(STX %g0, [%o0 + 24] ASINORMAL, add %o1, 8)
- add %o0, 32, %o0
-6: andncc %o1, 0x3f, %o3
-7: be,pn %xcc, 9f
-#ifdef __KERNEL__
- or %o4, ASI_BLK_OR, %g7
- wr %g7, %g0, %asi
- VISEntryHalf
-#else
- wr %g0, ASI_BLK_P, %asi
-#endif
- membar #StoreLoad | #StoreStore | #LoadStore
- fzero %f0
- andcc %o3, 0xc0, %o2
- and %o1, 0x3f, %o1
- fzero %f2
- andn %o3, 0xff, %o3
- faddd %f0, %f2, %f4
- fmuld %f0, %f2, %f6
- cmp %o2, 64
- faddd %f0, %f2, %f8
- fmuld %f0, %f2, %f10
- faddd %f0, %f2, %f12
- brz,pn %o2, 10f
- fmuld %f0, %f2, %f14
- be,pn %icc, 2f
- EXC(STBLK %f0, [%o0 + 0x00] ASIBLK, add %o3, %o2, add %o2, %o1,
%o2)
- cmp %o2, 128
- be,pn %icc, 2f
- EXC(STBLK %f0, [%o0 + 0x40] ASIBLK, add %o3, %o2, add %o2, %o1,
%o2; sub %o2, 64, %o2)
- EXC(STBLK %f0, [%o0 + 0x80] ASIBLK, add %o3, %o2, add %o2, %o1,
%o2; sub %o2, 128, %o2)
-2: brz,pn %o3, 12f
- add %o0, %o2, %o0
-10: EX(STBLK %f0, [%o0 + 0x00] ASIBLK, add %o3, %o1)
- EXC(STBLK %f0, [%o0 + 0x40] ASIBLK, add %o3, %o1, sub %o1, 64,
%o1)
- EXC(STBLK %f0, [%o0 + 0x80] ASIBLK, add %o3, %o1, sub %o1, 128,
%o1)
- EXC(STBLK %f0, [%o0 + 0xc0] ASIBLK, add %o3, %o1, sub %o1, 192,
%o1)
-11: subcc %o3, 256, %o3
- bne,pt %xcc, 10b
- add %o0, 256, %o0
-12:
-#ifdef __KERNEL__
- VISExitHalf
- wr %o4, 0x0, %asi
-#else
-#ifndef REGS_64BIT
- wr %g0, FPRS_FEF, %fprs
-#endif
-#endif
- membar #StoreLoad | #StoreStore
-9: andcc %o1, 0xf8, %o2
- be,pn %xcc, 13f
- andcc %o1, 7, %o1
-#ifdef __KERNEL__
-14: sethi %hi(13f), %o4
- srl %o2, 1, %o3
- sub %o4, %o3, %o4
- jmpl %o4 + %lo(13f), %g0
- add %o0, %o2, %o0
-#else
-14: rd %pc, %o4
- srl %o2, 1, %o3
- sub %o4, %o3, %o4
- jmpl %o4 + (13f - 14b), %g0
- add %o0, %o2, %o0
-#endif
-12: ZERO_BLOCKS(%o0, 0xc8, %g0)
- ZERO_BLOCKS(%o0, 0x88, %g0)
- ZERO_BLOCKS(%o0, 0x48, %g0)
- ZERO_BLOCKS(%o0, 0x08, %g0)
- EXT(12b,13f,VISbzerofixup_zb)
-13: be,pn %xcc, 8f
- andcc %o1, 4, %g0
- be,pn %xcc, 1f
- andcc %o1, 2, %g0
- EX(STW %g0, [%o0] ASINORMAL, and %o1, 7)
- add %o0, 4, %o0
-1: be,pn %xcc, 1f
- andcc %o1, 1, %g0
- EX(STH %g0, [%o0] ASINORMAL, and %o1, 3)
- add %o0, 2, %o0
-1: bne,a,pn %xcc, 8f
- EX(STB %g0, [%o0] ASINORMAL, add %g0, 1)
-8:
-#ifdef __KERNEL__
- wr %g5, %g0, %asi
-#endif
- retl
- RETL
-17: be,pn %xcc, 13b
- orcc %o1, 0, %g0
- be,pn %xcc, 0f
-8: add %o0, 1, %o0
- subcc %o1, 1, %o1
- bne,pt %xcc, 8b
- EX(STB %g0, [%o0 - 1] ASINORMAL, add %o1, 1)
-0:
-#ifdef __KERNEL__
- wr %g5, %g0, %asi
-#endif
- retl
- RETL
-
-#ifdef __KERNEL__
- .section .fixup
- .align 4
-VISbzerofixup_reto1:
- mov %o1, %o0
-VISbzerofixup_ret0:
- wr %g5, %g0, %asi
- retl
- wr %g0, 0, %fprs
-VISbzerofixup_ret1:
- and %o5, 0x30, %o5
- add %o5, %o1, %o5
- ba,pt %xcc, VISbzerofixup_ret0
- add %o0, %o5, %o0
-VISbzerofixup_ret2:
- and %o5, 0x20, %o5
- add %o5, %o1, %o5
- ba,pt %xcc, VISbzerofixup_ret0
- add %o0, %o5, %o0
-VISbzerofixup_zb:
- andcc %o1, 7, %o1
- sll %g2, 3, %g2
- add %o1, 256, %o1
- ba,pt %xcc, VISbzerofixup_ret0
- sub %o1, %g2, %o0
-#endif
diff -Nru a/arch/sparc64/lib/VISmemset.S b/arch/sparc64/lib/VISmemset.S
--- a/arch/sparc64/lib/VISmemset.S 2005-04-03 21:17:36 -07:00
+++ /dev/null Wed Dec 31 16:00:00 196900
@@ -1,240 +0,0 @@
-/* $Id: VISmemset.S,v 1.10 1999/12/23 17:02:16 jj Exp $
- * VISmemset.S: High speed memset operations utilizing the UltraSparc
- * Visual Instruction Set.
- *
- * Copyright (C) 1997 David S. Miller ([EMAIL PROTECTED])
- * Copyright (C) 1996, 1997, 1999 Jakub Jelinek ([EMAIL PROTECTED])
- */
-
-#include "VIS.h"
-
-#ifdef REGS_64BIT
-#define SET_BLOCKS(base, offset, source) \
- stx source, [base - offset - 0x18]; \
- stx source, [base - offset - 0x10]; \
- stx source, [base - offset - 0x08]; \
- stx source, [base - offset - 0x00];
-#else
-#define SET_BLOCKS(base, offset, source) \
- stw source, [base - offset - 0x18]; \
- stw source, [base - offset - 0x14]; \
- stw source, [base - offset - 0x10]; \
- stw source, [base - offset - 0x0c]; \
- stw source, [base - offset - 0x08]; \
- stw source, [base - offset - 0x04]; \
- stw source, [base - offset - 0x00]; \
- stw source, [base - offset + 0x04];
-#endif
-
-#ifndef __KERNEL__
-/* So that the brz,a,pt in memset doesn't have to get through PLT, here we
go... */
-#include "VISbzero.S"
-#endif
-
-#ifdef __KERNEL__
-#include <asm/visasm.h>
-#endif
-
- /* Well, memset is a lot easier to get right than bcopy... */
- .text
- .align 32
-#ifdef __KERNEL__
- .globl __memset
-__memset:
-#endif
- .globl memset
-memset:
-#ifndef __KERNEL__
- brz,a,pt %o1, bzero_private
- mov %o2, %o1
-#ifndef REGS_64BIT
- srl %o2, 0, %o2
-#endif
-#endif
- mov %o0, %o4
- cmp %o2, 7
- bleu,pn %xcc, 17f
- andcc %o0, 3, %g5
- be,pt %xcc, 4f
- and %o1, 0xff, %o1
- cmp %g5, 3
- be,pn %xcc, 2f
- stb %o1, [%o0 + 0x00]
- cmp %g5, 2
- be,pt %xcc, 2f
- stb %o1, [%o0 + 0x01]
- stb %o1, [%o0 + 0x02]
-2: sub %g5, 4, %g5
- sub %o0, %g5, %o0
- add %o2, %g5, %o2
-4: sllx %o1, 8, %g1
- andcc %o0, 4, %g0
- or %o1, %g1, %o1
- sllx %o1, 16, %g1
- or %o1, %g1, %o1
- be,pt %xcc, 2f
-#ifdef REGS_64BIT
- sllx %o1, 32, %g1
-#else
- cmp %o2, 128
-#endif
- stw %o1, [%o0]
- sub %o2, 4, %o2
- add %o0, 4, %o0
-2:
-#ifdef REGS_64BIT
- cmp %o2, 128
- or %o1, %g1, %o1
-#endif
- blu,pn %xcc, 9f
- andcc %o0, 0x38, %g5
- be,pn %icc, 6f
- mov 64, %o5
- andcc %o0, 8, %g0
- be,pn %icc, 1f
- sub %o5, %g5, %o5
-#ifdef REGS_64BIT
- stx %o1, [%o0]
-#else
- stw %o1, [%o0]
- stw %o1, [%o0 + 4]
-#endif
- add %o0, 8, %o0
-1: andcc %o5, 16, %g0
- be,pn %icc, 1f
- sub %o2, %o5, %o2
-#ifdef REGS_64BIT
- stx %o1, [%o0]
- stx %o1, [%o0 + 8]
-#else
- stw %o1, [%o0]
- stw %o1, [%o0 + 4]
- stw %o1, [%o0 + 8]
- stw %o1, [%o0 + 12]
-#endif
- add %o0, 16, %o0
-1: andcc %o5, 32, %g0
- be,pn %icc, 7f
- andncc %o2, 0x3f, %o3
-#ifdef REGS_64BIT
- stx %o1, [%o0]
- stx %o1, [%o0 + 8]
- stx %o1, [%o0 + 16]
- stx %o1, [%o0 + 24]
-#else
- stw %o1, [%o0]
- stw %o1, [%o0 + 4]
- stw %o1, [%o0 + 8]
- stw %o1, [%o0 + 12]
- stw %o1, [%o0 + 16]
- stw %o1, [%o0 + 20]
- stw %o1, [%o0 + 24]
- stw %o1, [%o0 + 28]
-#endif
- add %o0, 32, %o0
-7: be,pn %xcc, 9f
- nop
-#ifdef __KERNEL__
- VISEntryHalf
-#endif
- ldd [%o0 - 8], %f0
-18: rd %asi, %g2
- wr %g0, ASI_BLK_P, %asi
- membar #StoreStore | #LoadStore
- andcc %o3, 0xc0, %g5
- and %o2, 0x3f, %o2
- fmovd %f0, %f2
- fmovd %f0, %f4
- andn %o3, 0xff, %o3
- fmovd %f0, %f6
- cmp %g5, 64
- fmovd %f0, %f8
- fmovd %f0, %f10
- fmovd %f0, %f12
- brz,pn %g5, 10f
- fmovd %f0, %f14
- be,pn %icc, 2f
- stda %f0, [%o0 + 0x00] %asi
- cmp %g5, 128
- be,pn %icc, 2f
- stda %f0, [%o0 + 0x40] %asi
- stda %f0, [%o0 + 0x80] %asi
-2: brz,pn %o3, 12f
- add %o0, %g5, %o0
-10: stda %f0, [%o0 + 0x00] %asi
- stda %f0, [%o0 + 0x40] %asi
- stda %f0, [%o0 + 0x80] %asi
- stda %f0, [%o0 + 0xc0] %asi
-11: subcc %o3, 256, %o3
- bne,pt %xcc, 10b
- add %o0, 256, %o0
-12:
-#ifdef __KERNEL__
- wr %g2, %g0, %asi
- VISExitHalf
-#else
-#ifndef REGS_64BIT
- wr %g0, FPRS_FEF, %fprs
-#endif
-#endif
- membar #StoreLoad | #StoreStore
-9: andcc %o2, 0x78, %g5
- be,pn %xcc, 13f
- andcc %o2, 7, %o2
-#ifdef __KERNEL__
-14: srl %g5, 1, %o3
- sethi %hi(13f), %g3
- sub %g3, %o3, %g3
- jmpl %g3 + %lo(13f), %g0
- add %o0, %g5, %o0
-#else
-14: rd %pc, %g3
-#ifdef REGS_64BIT
- srl %g5, 1, %o3
- sub %g3, %o3, %g3
-#else
- sub %g3, %g5, %g3
-#endif
- jmpl %g3 + (13f - 14b), %g0
- add %o0, %g5, %o0
-#endif
-12: SET_BLOCKS(%o0, 0x68, %o1)
- SET_BLOCKS(%o0, 0x48, %o1)
- SET_BLOCKS(%o0, 0x28, %o1)
- SET_BLOCKS(%o0, 0x08, %o1)
-13: be,pn %xcc, 8f
- andcc %o2, 4, %g0
- be,pn %xcc, 1f
- andcc %o2, 2, %g0
- stw %o1, [%o0]
- add %o0, 4, %o0
-1: be,pn %xcc, 1f
- andcc %o2, 1, %g0
- sth %o1, [%o0]
- add %o0, 2, %o0
-1: bne,a,pn %xcc, 8f
- stb %o1, [%o0]
-8: retl
- mov %o4, %o0
-17: brz,pn %o2, 0f
-8: add %o0, 1, %o0
- subcc %o2, 1, %o2
- bne,pt %xcc, 8b
- stb %o1, [%o0 - 1]
-0: retl
- mov %o4, %o0
-6:
-#ifdef REGS_64BIT
- stx %o1, [%o0]
-#else
- stw %o1, [%o0]
- stw %o1, [%o0 + 4]
-#endif
- andncc %o2, 0x3f, %o3
- be,pn %xcc, 9b
- nop
-#ifdef __KERNEL__
- VISEntryHalf
-#endif
- ba,pt %xcc, 18b
- ldd [%o0], %f0
diff -Nru a/arch/sparc64/lib/atomic.S b/arch/sparc64/lib/atomic.S
--- a/arch/sparc64/lib/atomic.S 2005-04-03 21:17:36 -07:00
+++ b/arch/sparc64/lib/atomic.S 2005-04-03 21:17:36 -07:00
@@ -29,10 +29,10 @@
.globl atomic_add
.type atomic_add,#function
atomic_add: /* %o0 = increment, %o1 = atomic_ptr */
-1: lduw [%o1], %g5
- add %g5, %o0, %g7
- cas [%o1], %g5, %g7
- cmp %g5, %g7
+1: lduw [%o1], %g1
+ add %g1, %o0, %g7
+ cas [%o1], %g1, %g7
+ cmp %g1, %g7
bne,pn %icc, 1b
nop
retl
@@ -42,10 +42,10 @@
.globl atomic_sub
.type atomic_sub,#function
atomic_sub: /* %o0 = decrement, %o1 = atomic_ptr */
-1: lduw [%o1], %g5
- sub %g5, %o0, %g7
- cas [%o1], %g5, %g7
- cmp %g5, %g7
+1: lduw [%o1], %g1
+ sub %g1, %o0, %g7
+ cas [%o1], %g1, %g7
+ cmp %g1, %g7
bne,pn %icc, 1b
nop
retl
@@ -56,10 +56,10 @@
.type atomic_add_ret,#function
atomic_add_ret: /* %o0 = increment, %o1 = atomic_ptr */
ATOMIC_PRE_BARRIER
-1: lduw [%o1], %g5
- add %g5, %o0, %g7
- cas [%o1], %g5, %g7
- cmp %g5, %g7
+1: lduw [%o1], %g1
+ add %g1, %o0, %g7
+ cas [%o1], %g1, %g7
+ cmp %g1, %g7
bne,pn %icc, 1b
add %g7, %o0, %g7
ATOMIC_POST_BARRIER
@@ -71,10 +71,10 @@
.type atomic_sub_ret,#function
atomic_sub_ret: /* %o0 = decrement, %o1 = atomic_ptr */
ATOMIC_PRE_BARRIER
-1: lduw [%o1], %g5
- sub %g5, %o0, %g7
- cas [%o1], %g5, %g7
- cmp %g5, %g7
+1: lduw [%o1], %g1
+ sub %g1, %o0, %g7
+ cas [%o1], %g1, %g7
+ cmp %g1, %g7
bne,pn %icc, 1b
sub %g7, %o0, %g7
ATOMIC_POST_BARRIER
@@ -85,10 +85,10 @@
.globl atomic64_add
.type atomic64_add,#function
atomic64_add: /* %o0 = increment, %o1 = atomic_ptr */
-1: ldx [%o1], %g5
- add %g5, %o0, %g7
- casx [%o1], %g5, %g7
- cmp %g5, %g7
+1: ldx [%o1], %g1
+ add %g1, %o0, %g7
+ casx [%o1], %g1, %g7
+ cmp %g1, %g7
bne,pn %xcc, 1b
nop
retl
@@ -98,10 +98,10 @@
.globl atomic64_sub
.type atomic64_sub,#function
atomic64_sub: /* %o0 = decrement, %o1 = atomic_ptr */
-1: ldx [%o1], %g5
- sub %g5, %o0, %g7
- casx [%o1], %g5, %g7
- cmp %g5, %g7
+1: ldx [%o1], %g1
+ sub %g1, %o0, %g7
+ casx [%o1], %g1, %g7
+ cmp %g1, %g7
bne,pn %xcc, 1b
nop
retl
@@ -112,10 +112,10 @@
.type atomic64_add_ret,#function
atomic64_add_ret: /* %o0 = increment, %o1 = atomic_ptr */
ATOMIC_PRE_BARRIER
-1: ldx [%o1], %g5
- add %g5, %o0, %g7
- casx [%o1], %g5, %g7
- cmp %g5, %g7
+1: ldx [%o1], %g1
+ add %g1, %o0, %g7
+ casx [%o1], %g1, %g7
+ cmp %g1, %g7
bne,pn %xcc, 1b
add %g7, %o0, %g7
ATOMIC_POST_BARRIER
@@ -127,10 +127,10 @@
.type atomic64_sub_ret,#function
atomic64_sub_ret: /* %o0 = decrement, %o1 = atomic_ptr */
ATOMIC_PRE_BARRIER
-1: ldx [%o1], %g5
- sub %g5, %o0, %g7
- casx [%o1], %g5, %g7
- cmp %g5, %g7
+1: ldx [%o1], %g1
+ sub %g1, %o0, %g7
+ casx [%o1], %g1, %g7
+ cmp %g1, %g7
bne,pn %xcc, 1b
sub %g7, %o0, %g7
ATOMIC_POST_BARRIER
diff -Nru a/arch/sparc64/lib/bzero.S b/arch/sparc64/lib/bzero.S
--- /dev/null Wed Dec 31 16:00:00 196900
+++ b/arch/sparc64/lib/bzero.S 2005-04-03 21:17:36 -07:00
@@ -0,0 +1,158 @@
+/* bzero.S: Simple prefetching memset, bzero, and clear_user
+ * implementations.
+ *
+ * Copyright (C) 2005 David S. Miller <[EMAIL PROTECTED]>
+ */
+
+ .text
+
+ .globl __memset
+ .type __memset, #function
+__memset: /* %o0=buf, %o1=pat, %o2=len */
+
+ .globl memset
+ .type memset, #function
+memset: /* %o0=buf, %o1=pat, %o2=len */
+ and %o1, 0xff, %o3
+ mov %o2, %o1
+ sllx %o3, 8, %g1
+ or %g1, %o3, %o2
+ sllx %o2, 16, %g1
+ or %g1, %o2, %o2
+ sllx %o2, 32, %g1
+ ba,pt %xcc, 1f
+ or %g1, %o2, %o2
+
+ .globl __bzero
+ .type __bzero, #function
+__bzero: /* %o0=buf, %o1=len */
+ clr %o2
+1: mov %o0, %o3
+ brz,pn %o1, __bzero_done
+ cmp %o1, 16
+ bl,pn %icc, __bzero_tiny
+ prefetch [%o0 + 0x000], #n_writes
+ andcc %o0, 0x3, %g0
+ be,pt %icc, 2f
+1: stb %o2, [%o0 + 0x00]
+ add %o0, 1, %o0
+ andcc %o0, 0x3, %g0
+ bne,pn %icc, 1b
+ sub %o1, 1, %o1
+2: andcc %o0, 0x7, %g0
+ be,pt %icc, 3f
+ stw %o2, [%o0 + 0x00]
+ sub %o1, 4, %o1
+ add %o0, 4, %o0
+3: and %o1, 0x38, %g1
+ cmp %o1, 0x40
+ andn %o1, 0x3f, %o4
+ bl,pn %icc, 5f
+ and %o1, 0x7, %o1
+ prefetch [%o0 + 0x040], #n_writes
+ prefetch [%o0 + 0x080], #n_writes
+ prefetch [%o0 + 0x0c0], #n_writes
+ prefetch [%o0 + 0x100], #n_writes
+ prefetch [%o0 + 0x140], #n_writes
+4: prefetch [%o0 + 0x180], #n_writes
+ stx %o2, [%o0 + 0x00]
+ stx %o2, [%o0 + 0x08]
+ stx %o2, [%o0 + 0x10]
+ stx %o2, [%o0 + 0x18]
+ stx %o2, [%o0 + 0x20]
+ stx %o2, [%o0 + 0x28]
+ stx %o2, [%o0 + 0x30]
+ stx %o2, [%o0 + 0x38]
+ subcc %o4, 0x40, %o4
+ bne,pt %icc, 4b
+ add %o0, 0x40, %o0
+ brz,pn %g1, 6f
+ nop
+5: stx %o2, [%o0 + 0x00]
+ subcc %g1, 8, %g1
+ bne,pt %icc, 5b
+ add %o0, 0x8, %o0
+6: brz,pt %o1, __bzero_done
+ nop
+__bzero_tiny:
+1: stb %o2, [%o0 + 0x00]
+ subcc %o1, 1, %o1
+ bne,pt %icc, 1b
+ add %o0, 1, %o0
+__bzero_done:
+ retl
+ mov %o3, %o0
+ .size __bzero, .-__bzero
+ .size __memset, .-__memset
+ .size memset, .-memset
+
+#define EX_ST(x,y) \
+98: x,y; \
+ .section .fixup; \
+ .align 4; \
+99: retl; \
+ mov %o1, %o0; \
+ .section __ex_table; \
+ .align 4; \
+ .word 98b, 99b; \
+ .text; \
+ .align 4;
+
+ .globl __bzero_noasi
+ .type __bzero_noasi, #function
+__bzero_noasi: /* %o0=buf, %o1=len */
+ brz,pn %o1, __bzero_noasi_done
+ cmp %o1, 16
+ bl,pn %icc, __bzero_noasi_tiny
+ EX_ST(prefetcha [%o0 + 0x00] %asi, #n_writes)
+ andcc %o0, 0x3, %g0
+ be,pt %icc, 2f
+1: EX_ST(stba %g0, [%o0 + 0x00] %asi)
+ add %o0, 1, %o0
+ andcc %o0, 0x3, %g0
+ bne,pn %icc, 1b
+ sub %o1, 1, %o1
+2: andcc %o0, 0x7, %g0
+ be,pt %icc, 3f
+ EX_ST(stwa %g0, [%o0 + 0x00] %asi)
+ sub %o1, 4, %o1
+ add %o0, 4, %o0
+3: and %o1, 0x38, %g1
+ cmp %o1, 0x40
+ andn %o1, 0x3f, %o4
+ bl,pn %icc, 5f
+ and %o1, 0x7, %o1
+ EX_ST(prefetcha [%o0 + 0x040] %asi, #n_writes)
+ EX_ST(prefetcha [%o0 + 0x080] %asi, #n_writes)
+ EX_ST(prefetcha [%o0 + 0x0c0] %asi, #n_writes)
+ EX_ST(prefetcha [%o0 + 0x100] %asi, #n_writes)
+ EX_ST(prefetcha [%o0 + 0x140] %asi, #n_writes)
+4: EX_ST(prefetcha [%o0 + 0x180] %asi, #n_writes)
+ EX_ST(stxa %g0, [%o0 + 0x00] %asi)
+ EX_ST(stxa %g0, [%o0 + 0x08] %asi)
+ EX_ST(stxa %g0, [%o0 + 0x10] %asi)
+ EX_ST(stxa %g0, [%o0 + 0x18] %asi)
+ EX_ST(stxa %g0, [%o0 + 0x20] %asi)
+ EX_ST(stxa %g0, [%o0 + 0x28] %asi)
+ EX_ST(stxa %g0, [%o0 + 0x30] %asi)
+ EX_ST(stxa %g0, [%o0 + 0x38] %asi)
+ subcc %o4, 0x40, %o4
+ bne,pt %icc, 4b
+ add %o0, 0x40, %o0
+ brz,pn %g1, 6f
+ nop
+5: EX_ST(stxa %g0, [%o0 + 0x00] %asi)
+ subcc %g1, 8, %g1
+ bne,pt %icc, 5b
+ add %o0, 0x8, %o0
+6: brz,pt %o1, __bzero_noasi_done
+ nop
+__bzero_noasi_tiny:
+1: EX_ST(stba %g0, [%o0 + 0x00] %asi)
+ subcc %o1, 1, %o1
+ bne,pt %icc, 1b
+ add %o0, 1, %o0
+__bzero_noasi_done:
+ retl
+ clr %o0
+ .size __bzero_noasi, .-__bzero_noasi
diff -Nru a/arch/sparc64/lib/debuglocks.c b/arch/sparc64/lib/debuglocks.c
--- a/arch/sparc64/lib/debuglocks.c 2005-04-03 21:17:36 -07:00
+++ b/arch/sparc64/lib/debuglocks.c 2005-04-03 21:17:36 -07:00
@@ -138,15 +138,15 @@
}
/* Try once to increment the counter. */
__asm__ __volatile__(
-" ldx [%0], %%g5\n"
-" brlz,a,pn %%g5, 2f\n"
+" ldx [%0], %%g1\n"
+" brlz,a,pn %%g1, 2f\n"
" mov 1, %0\n"
-" add %%g5, 1, %%g7\n"
-" casx [%0], %%g5, %%g7\n"
-" sub %%g5, %%g7, %0\n"
+" add %%g1, 1, %%g7\n"
+" casx [%0], %%g1, %%g7\n"
+" sub %%g1, %%g7, %0\n"
"2:" : "=r" (val)
: "0" (&(rw->lock))
- : "g5", "g7", "memory");
+ : "g1", "g7", "memory");
membar("#StoreLoad | #StoreStore");
if (val)
goto wlock_again;
@@ -173,14 +173,14 @@
/* Spin trying to decrement the counter using casx. */
__asm__ __volatile__(
" membar #StoreLoad | #LoadLoad\n"
-" ldx [%0], %%g5\n"
-" sub %%g5, 1, %%g7\n"
-" casx [%0], %%g5, %%g7\n"
+" ldx [%0], %%g1\n"
+" sub %%g1, 1, %%g7\n"
+" casx [%0], %%g1, %%g7\n"
" membar #StoreLoad | #StoreStore\n"
-" sub %%g5, %%g7, %0\n"
+" sub %%g1, %%g7, %0\n"
: "=r" (val)
: "0" (&(rw->lock))
- : "g5", "g7", "memory");
+ : "g1", "g7", "memory");
if (val) {
if (!--stuck) {
if (shown++ <= 2)
@@ -216,17 +216,17 @@
__asm__ __volatile__(
" mov 1, %%g3\n"
" sllx %%g3, 63, %%g3\n"
-" ldx [%0], %%g5\n"
-" brlz,pn %%g5, 1f\n"
-" or %%g5, %%g3, %%g7\n"
-" casx [%0], %%g5, %%g7\n"
+" ldx [%0], %%g1\n"
+" brlz,pn %%g1, 1f\n"
+" or %%g1, %%g3, %%g7\n"
+" casx [%0], %%g1, %%g7\n"
" membar #StoreLoad | #StoreStore\n"
" ba,pt %%xcc, 2f\n"
-" sub %%g5, %%g7, %0\n"
+" sub %%g1, %%g7, %0\n"
"1: mov 1, %0\n"
"2:" : "=r" (val)
: "0" (&(rw->lock))
- : "g3", "g5", "g7", "memory");
+ : "g3", "g1", "g7", "memory");
if (val) {
/* We couldn't get the write bit. */
if (!--stuck) {
@@ -248,15 +248,15 @@
__asm__ __volatile__(
" mov 1, %%g3\n"
" sllx %%g3, 63, %%g3\n"
-"1: ldx [%0], %%g5\n"
-" andn %%g5, %%g3, %%g7\n"
-" casx [%0], %%g5, %%g7\n"
-" cmp %%g5, %%g7\n"
+"1: ldx [%0], %%g1\n"
+" andn %%g1, %%g3, %%g7\n"
+" casx [%0], %%g1, %%g7\n"
+" cmp %%g1, %%g7\n"
" bne,pn %%xcc, 1b\n"
" membar #StoreLoad | #StoreStore"
: /* no outputs */
: "r" (&(rw->lock))
- : "g3", "g5", "g7", "cc", "memory");
+ : "g3", "g1", "g7", "cc", "memory");
while(rw->lock != 0) {
if (!--stuck) {
if (shown++ <= 2)
@@ -294,14 +294,14 @@
" membar #StoreLoad | #LoadLoad\n"
" mov 1, %%g3\n"
" sllx %%g3, 63, %%g3\n"
-" ldx [%0], %%g5\n"
-" andn %%g5, %%g3, %%g7\n"
-" casx [%0], %%g5, %%g7\n"
+" ldx [%0], %%g1\n"
+" andn %%g1, %%g3, %%g7\n"
+" casx [%0], %%g1, %%g7\n"
" membar #StoreLoad | #StoreStore\n"
-" sub %%g5, %%g7, %0\n"
+" sub %%g1, %%g7, %0\n"
: "=r" (val)
: "0" (&(rw->lock))
- : "g3", "g5", "g7", "memory");
+ : "g3", "g1", "g7", "memory");
if (val) {
if (!--stuck) {
if (shown++ <= 2)
@@ -323,17 +323,17 @@
__asm__ __volatile__(
" mov 1, %%g3\n"
" sllx %%g3, 63, %%g3\n"
-" ldx [%0], %%g5\n"
-" brlz,pn %%g5, 1f\n"
-" or %%g5, %%g3, %%g7\n"
-" casx [%0], %%g5, %%g7\n"
+" ldx [%0], %%g1\n"
+" brlz,pn %%g1, 1f\n"
+" or %%g1, %%g3, %%g7\n"
+" casx [%0], %%g1, %%g7\n"
" membar #StoreLoad | #StoreStore\n"
" ba,pt %%xcc, 2f\n"
-" sub %%g5, %%g7, %0\n"
+" sub %%g1, %%g7, %0\n"
"1: mov 1, %0\n"
"2:" : "=r" (val)
: "0" (&(rw->lock))
- : "g3", "g5", "g7", "memory");
+ : "g3", "g1", "g7", "memory");
if (val) {
put_cpu();
@@ -347,15 +347,15 @@
__asm__ __volatile__(
" mov 1, %%g3\n"
" sllx %%g3, 63, %%g3\n"
-"1: ldx [%0], %%g5\n"
-" andn %%g5, %%g3, %%g7\n"
-" casx [%0], %%g5, %%g7\n"
-" cmp %%g5, %%g7\n"
+"1: ldx [%0], %%g1\n"
+" andn %%g1, %%g3, %%g7\n"
+" casx [%0], %%g1, %%g7\n"
+" cmp %%g1, %%g7\n"
" bne,pn %%xcc, 1b\n"
" membar #StoreLoad | #StoreStore"
: /* no outputs */
: "r" (&(rw->lock))
- : "g3", "g5", "g7", "cc", "memory");
+ : "g3", "g1", "g7", "cc", "memory");
put_cpu();
diff -Nru a/arch/sparc64/lib/dec_and_lock.S b/arch/sparc64/lib/dec_and_lock.S
--- a/arch/sparc64/lib/dec_and_lock.S 2005-04-03 21:17:36 -07:00
+++ b/arch/sparc64/lib/dec_and_lock.S 2005-04-03 21:17:36 -07:00
@@ -27,12 +27,12 @@
.globl _atomic_dec_and_lock
_atomic_dec_and_lock: /* %o0 = counter, %o1 = lock */
-loop1: lduw [%o0], %g5
- subcc %g5, 1, %g7
+loop1: lduw [%o0], %g2
+ subcc %g2, 1, %g7
be,pn %icc, start_to_zero
nop
-nzero: cas [%o0], %g5, %g7
- cmp %g5, %g7
+nzero: cas [%o0], %g2, %g7
+ cmp %g2, %g7
bne,pn %icc, loop1
mov 0, %g1
@@ -50,13 +50,13 @@
ldstub [%o1], %g3
brnz,pn %g3, spin_on_lock
membar #StoreLoad | #StoreStore
-loop2: cas [%o0], %g5, %g7 /* ASSERT(g7 == 0) */
- cmp %g5, %g7
+loop2: cas [%o0], %g2, %g7 /* ASSERT(g7 == 0) */
+ cmp %g2, %g7
be,pt %icc, out
mov 1, %g1
- lduw [%o0], %g5
- subcc %g5, 1, %g7
+ lduw [%o0], %g2
+ subcc %g2, 1, %g7
be,pn %icc, loop2
nop
membar #StoreStore | #LoadStore
diff -Nru a/arch/sparc64/lib/mcount.S b/arch/sparc64/lib/mcount.S
--- a/arch/sparc64/lib/mcount.S 2005-04-03 21:17:36 -07:00
+++ b/arch/sparc64/lib/mcount.S 2005-04-03 21:17:36 -07:00
@@ -38,22 +38,22 @@
* Check whether %sp is dangerously low.
*/
ldub [%g6 + TI_FPDEPTH], %g1
- srl %g1, 1, %g5
- add %g5, 1, %g5
- sllx %g5, 8, %g5 ! each fpregs frame is
256b
- add %g5, 192, %g5
- add %g6, %g5, %g5 ! where does
task_struct+frame end?
- sub %g5, STACK_BIAS, %g5
- cmp %sp, %g5
+ srl %g1, 1, %g3
+ add %g3, 1, %g3
+ sllx %g3, 8, %g3 ! each fpregs frame is
256b
+ add %g3, 192, %g3
+ add %g6, %g3, %g3 ! where does
task_struct+frame end?
+ sub %g3, STACK_BIAS, %g3
+ cmp %sp, %g3
bg,pt %xcc, 1f
- sethi %hi(panicstring), %g5
+ sethi %hi(panicstring), %g3
sethi %hi(ovstack), %g7 ! cant move to panic
stack fast enough
or %g7, %lo(ovstack), %g7
add %g7, OVSTACKSIZE, %g7
sub %g7, STACK_BIAS, %g7
mov %g7, %sp
call prom_printf
- or %g5, %lo(panicstring), %o0
+ or %g3, %lo(panicstring), %o0
call prom_halt
nop
#endif
diff -Nru a/arch/sparc64/lib/memcmp.S b/arch/sparc64/lib/memcmp.S
--- a/arch/sparc64/lib/memcmp.S 2005-04-03 21:17:36 -07:00
+++ b/arch/sparc64/lib/memcmp.S 2005-04-03 21:17:36 -07:00
@@ -13,12 +13,12 @@
cmp %o2, 0 ! IEU1 Group
loop: be,pn %icc, ret_0 ! CTI
nop ! IEU0
- ldub [%o0], %g5 ! LSU Group
+ ldub [%o0], %g7 ! LSU Group
ldub [%o1], %g3 ! LSU Group
sub %o2, 1, %o2 ! IEU0
add %o0, 1, %o0 ! IEU1
add %o1, 1, %o1 ! IEU0 Group
- subcc %g5, %g3, %g3 ! IEU1 Group
+ subcc %g7, %g3, %g3 ! IEU1 Group
be,pt %icc, loop ! CTI
cmp %o2, 0 ! IEU1 Group
diff -Nru a/arch/sparc64/lib/memmove.S b/arch/sparc64/lib/memmove.S
--- a/arch/sparc64/lib/memmove.S 2005-04-03 21:17:36 -07:00
+++ b/arch/sparc64/lib/memmove.S 2005-04-03 21:17:36 -07:00
@@ -12,17 +12,17 @@
mov %o0, %g1
cmp %o0, %o1
bleu,pt %xcc, memcpy
- add %o1, %o2, %g5
- cmp %g5, %o0
+ add %o1, %o2, %g7
+ cmp %g7, %o0
bleu,pt %xcc, memcpy
add %o0, %o2, %o5
- sub %g5, 1, %o1
+ sub %g7, 1, %o1
sub %o5, 1, %o0
-1: ldub [%o1], %g5
+1: ldub [%o1], %g7
subcc %o2, 1, %o2
sub %o1, 1, %o1
- stb %g5, [%o0]
+ stb %g7, [%o0]
bne,pt %icc, 1b
sub %o0, 1, %o0
diff -Nru a/arch/sparc64/lib/memscan.S b/arch/sparc64/lib/memscan.S
--- a/arch/sparc64/lib/memscan.S 2005-04-03 21:17:36 -07:00
+++ b/arch/sparc64/lib/memscan.S 2005-04-03 21:17:36 -07:00
@@ -52,43 +52,43 @@
andcc %o5, 0xff, %g0
add %o0, -5, %g2
ba,pt %xcc, 3f
- srlx %o5, 32, %g5
+ srlx %o5, 32, %g7
-2: srlx %o5, 8, %g5
+2: srlx %o5, 8, %g7
be,pn %icc, 1f
add %o0, -8, %g2
- andcc %g5, 0xff, %g0
- srlx %g5, 8, %g5
+ andcc %g7, 0xff, %g0
+ srlx %g7, 8, %g7
be,pn %icc, 1f
inc %g2
- andcc %g5, 0xff, %g0
+ andcc %g7, 0xff, %g0
- srlx %g5, 8, %g5
+ srlx %g7, 8, %g7
be,pn %icc, 1f
inc %g2
- andcc %g5, 0xff, %g0
- srlx %g5, 8, %g5
+ andcc %g7, 0xff, %g0
+ srlx %g7, 8, %g7
be,pn %icc, 1f
inc %g2
andcc %g3, %o3, %g0
be,a,pn %icc, 2f
mov %o0, %g2
-3: andcc %g5, 0xff, %g0
- srlx %g5, 8, %g5
+3: andcc %g7, 0xff, %g0
+ srlx %g7, 8, %g7
be,pn %icc, 1f
inc %g2
- andcc %g5, 0xff, %g0
- srlx %g5, 8, %g5
+ andcc %g7, 0xff, %g0
+ srlx %g7, 8, %g7
be,pn %icc, 1f
inc %g2
- andcc %g5, 0xff, %g0
- srlx %g5, 8, %g5
+ andcc %g7, 0xff, %g0
+ srlx %g7, 8, %g7
be,pn %icc, 1f
inc %g2
- andcc %g5, 0xff, %g0
- srlx %g5, 8, %g5
+ andcc %g7, 0xff, %g0
+ srlx %g7, 8, %g7
be,pn %icc, 1f
inc %g2
diff -Nru a/arch/sparc64/lib/strlen.S b/arch/sparc64/lib/strlen.S
--- a/arch/sparc64/lib/strlen.S 2005-04-03 21:17:36 -07:00
+++ b/arch/sparc64/lib/strlen.S 2005-04-03 21:17:36 -07:00
@@ -48,16 +48,16 @@
add %o0, 4, %o0
/* Check every byte. */
- srl %o5, 24, %g5
- andcc %g5, 0xff, %g0
+ srl %o5, 24, %g7
+ andcc %g7, 0xff, %g0
be,pn %icc, 1f
add %o0, -4, %o4
- srl %o5, 16, %g5
- andcc %g5, 0xff, %g0
+ srl %o5, 16, %g7
+ andcc %g7, 0xff, %g0
be,pn %icc, 1f
add %o4, 1, %o4
- srl %o5, 8, %g5
- andcc %g5, 0xff, %g0
+ srl %o5, 8, %g7
+ andcc %g7, 0xff, %g0
be,pn %icc, 1f
add %o4, 1, %o4
andcc %o5, 0xff, %g0
diff -Nru a/arch/sparc64/lib/strlen_user.S b/arch/sparc64/lib/strlen_user.S
--- a/arch/sparc64/lib/strlen_user.S 2005-04-03 21:17:36 -07:00
+++ b/arch/sparc64/lib/strlen_user.S 2005-04-03 21:17:36 -07:00
@@ -54,16 +54,16 @@
ba,a,pt %xcc, 1f
/* Check every byte. */
-82: srl %o5, 24, %g5
- andcc %g5, 0xff, %g0
+82: srl %o5, 24, %g7
+ andcc %g7, 0xff, %g0
be,pn %icc, 1f
add %o0, -3, %o4
- srl %o5, 16, %g5
- andcc %g5, 0xff, %g0
+ srl %o5, 16, %g7
+ andcc %g7, 0xff, %g0
be,pn %icc, 1f
add %o4, 1, %o4
- srl %o5, 8, %g5
- andcc %g5, 0xff, %g0
+ srl %o5, 8, %g7
+ andcc %g7, 0xff, %g0
be,pn %icc, 1f
add %o4, 1, %o4
andcc %o5, 0xff, %g0
diff -Nru a/arch/sparc64/lib/xor.S b/arch/sparc64/lib/xor.S
--- a/arch/sparc64/lib/xor.S 2005-04-03 21:17:36 -07:00
+++ b/arch/sparc64/lib/xor.S 2005-04-03 21:17:36 -07:00
@@ -248,7 +248,7 @@
.globl xor_vis_5
.type xor_vis_5,#function
xor_vis_5:
- mov %o5, %g5
+ save %sp, -192, %sp
rd %fprs, %o5
andcc %o5, FPRS_FEF|FPRS_DU, %g0
be,pt %icc, 0f
@@ -256,61 +256,60 @@
jmpl %g1 + %lo(VISenter), %g7
add %g7, 8, %g7
0: wr %g0, FPRS_FEF, %fprs
- mov %g5, %o5
rd %asi, %g1
wr %g0, ASI_BLK_P, %asi
membar #LoadStore|#StoreLoad|#StoreStore
- sub %o0, 64, %o0
- ldda [%o1] %asi, %f0
- ldda [%o2] %asi, %f16
+ sub %i0, 64, %i0
+ ldda [%i1] %asi, %f0
+ ldda [%i2] %asi, %f16
-5: ldda [%o3] %asi, %f32
+5: ldda [%i3] %asi, %f32
fxor %f0, %f16, %f48
fxor %f2, %f18, %f50
- add %o1, 64, %o1
+ add %i1, 64, %i1
fxor %f4, %f20, %f52
fxor %f6, %f22, %f54
- add %o2, 64, %o2
+ add %i2, 64, %i2
fxor %f8, %f24, %f56
fxor %f10, %f26, %f58
fxor %f12, %f28, %f60
fxor %f14, %f30, %f62
- ldda [%o4] %asi, %f16
+ ldda [%i4] %asi, %f16
fxor %f48, %f32, %f48
fxor %f50, %f34, %f50
fxor %f52, %f36, %f52
fxor %f54, %f38, %f54
- add %o3, 64, %o3
+ add %i3, 64, %i3
fxor %f56, %f40, %f56
fxor %f58, %f42, %f58
fxor %f60, %f44, %f60
fxor %f62, %f46, %f62
- ldda [%o5] %asi, %f32
+ ldda [%i5] %asi, %f32
fxor %f48, %f16, %f48
fxor %f50, %f18, %f50
- add %o4, 64, %o4
+ add %i4, 64, %i4
fxor %f52, %f20, %f52
fxor %f54, %f22, %f54
- add %o5, 64, %o5
+ add %i5, 64, %i5
fxor %f56, %f24, %f56
fxor %f58, %f26, %f58
fxor %f60, %f28, %f60
fxor %f62, %f30, %f62
- ldda [%o1] %asi, %f0
+ ldda [%i1] %asi, %f0
fxor %f48, %f32, %f48
fxor %f50, %f34, %f50
fxor %f52, %f36, %f52
fxor %f54, %f38, %f54
fxor %f56, %f40, %f56
fxor %f58, %f42, %f58
- subcc %o0, 64, %o0
+ subcc %i0, 64, %i0
fxor %f60, %f44, %f60
fxor %f62, %f46, %f62
- stda %f48, [%o1 - 64] %asi
+ stda %f48, [%i1 - 64] %asi
bne,pt %xcc, 5b
- ldda [%o2] %asi, %f16
+ ldda [%i2] %asi, %f16
- ldda [%o3] %asi, %f32
+ ldda [%i3] %asi, %f32
fxor %f0, %f16, %f48
fxor %f2, %f18, %f50
fxor %f4, %f20, %f52
@@ -319,7 +318,7 @@
fxor %f10, %f26, %f58
fxor %f12, %f28, %f60
fxor %f14, %f30, %f62
- ldda [%o4] %asi, %f16
+ ldda [%i4] %asi, %f16
fxor %f48, %f32, %f48
fxor %f50, %f34, %f50
fxor %f52, %f36, %f52
@@ -328,7 +327,7 @@
fxor %f58, %f42, %f58
fxor %f60, %f44, %f60
fxor %f62, %f46, %f62
- ldda [%o5] %asi, %f32
+ ldda [%i5] %asi, %f32
fxor %f48, %f16, %f48
fxor %f50, %f18, %f50
fxor %f52, %f20, %f52
@@ -346,9 +345,10 @@
fxor %f58, %f42, %f58
fxor %f60, %f44, %f60
fxor %f62, %f46, %f62
- stda %f48, [%o1] %asi
+ stda %f48, [%i1] %asi
membar #Sync|#StoreStore|#StoreLoad
wr %g1, %g0, %asi
- retl
- wr %g0, 0, %fprs
+ wr %g0, 0, %fprs
+ ret
+ restore
.size xor_vis_5, .-xor_vis_5
diff -Nru a/arch/sparc64/mm/init.c b/arch/sparc64/mm/init.c
--- a/arch/sparc64/mm/init.c 2005-04-03 21:17:36 -07:00
+++ b/arch/sparc64/mm/init.c 2005-04-03 21:17:36 -07:00
@@ -132,15 +132,15 @@
mask = (mask << 24) | (1UL << PG_dcache_dirty);
__asm__ __volatile__("1:\n\t"
"ldx [%2], %%g7\n\t"
- "and %%g7, %1, %%g5\n\t"
- "or %%g5, %0, %%g5\n\t"
- "casx [%2], %%g7, %%g5\n\t"
- "cmp %%g7, %%g5\n\t"
+ "and %%g7, %1, %%g1\n\t"
+ "or %%g1, %0, %%g1\n\t"
+ "casx [%2], %%g7, %%g1\n\t"
+ "cmp %%g7, %%g1\n\t"
"bne,pn %%xcc, 1b\n\t"
" membar #StoreLoad | #StoreStore"
: /* no outputs */
: "r" (mask), "r" (non_cpu_bits), "r"
(&page->flags)
- : "g5", "g7");
+ : "g1", "g7");
}
static __inline__ void clear_dcache_dirty_cpu(struct page *page, unsigned long
cpu)
@@ -150,20 +150,20 @@
__asm__ __volatile__("! test_and_clear_dcache_dirty\n"
"1:\n\t"
"ldx [%2], %%g7\n\t"
- "srlx %%g7, 24, %%g5\n\t"
- "and %%g5, %3, %%g5\n\t"
- "cmp %%g5, %0\n\t"
+ "srlx %%g7, 24, %%g1\n\t"
+ "and %%g1, %3, %%g1\n\t"
+ "cmp %%g1, %0\n\t"
"bne,pn %%icc, 2f\n\t"
- " andn %%g7, %1, %%g5\n\t"
- "casx [%2], %%g7, %%g5\n\t"
- "cmp %%g7, %%g5\n\t"
+ " andn %%g7, %1, %%g1\n\t"
+ "casx [%2], %%g7, %%g1\n\t"
+ "cmp %%g7, %%g1\n\t"
"bne,pn %%xcc, 1b\n\t"
" membar #StoreLoad | #StoreStore\n"
"2:"
: /* no outputs */
: "r" (cpu), "r" (mask), "r" (&page->flags),
"i" (NR_CPUS - 1UL)
- : "g5", "g7");
+ : "g1", "g7");
}
extern void __update_mmu_cache(unsigned long mmu_context_hw, unsigned long
address, pte_t pte, int code);
diff -Nru a/include/asm-sparc64/system.h b/include/asm-sparc64/system.h
--- a/include/asm-sparc64/system.h 2005-04-03 21:17:36 -07:00
+++ b/include/asm-sparc64/system.h 2005-04-03 21:17:36 -07:00
@@ -226,37 +226,41 @@
} \
} while(0)
-static __inline__ unsigned long xchg32(__volatile__ unsigned int *m, unsigned
int val)
+static inline unsigned long xchg32(__volatile__ unsigned int *m, unsigned int
val)
{
+ unsigned long tmp1, tmp2;
+
__asm__ __volatile__(
" membar #StoreLoad | #LoadLoad\n"
-" mov %0, %%g5\n"
-"1: lduw [%2], %%g7\n"
-" cas [%2], %%g7, %0\n"
-" cmp %%g7, %0\n"
+" mov %0, %1\n"
+"1: lduw [%4], %2\n"
+" cas [%4], %2, %0\n"
+" cmp %2, %0\n"
" bne,a,pn %%icc, 1b\n"
-" mov %%g5, %0\n"
+" mov %1, %0\n"
" membar #StoreLoad | #StoreStore\n"
- : "=&r" (val)
+ : "=&r" (val), "=&r" (tmp1), "=&r" (tmp2)
: "0" (val), "r" (m)
- : "g5", "g7", "cc", "memory");
+ : "cc", "memory");
return val;
}
-static __inline__ unsigned long xchg64(__volatile__ unsigned long *m, unsigned
long val)
+static inline unsigned long xchg64(__volatile__ unsigned long *m, unsigned
long val)
{
+ unsigned long tmp1, tmp2;
+
__asm__ __volatile__(
" membar #StoreLoad | #LoadLoad\n"
-" mov %0, %%g5\n"
-"1: ldx [%2], %%g7\n"
-" casx [%2], %%g7, %0\n"
-" cmp %%g7, %0\n"
+" mov %0, %1\n"
+"1: ldx [%4], %2\n"
+" casx [%4], %2, %0\n"
+" cmp %2, %0\n"
" bne,a,pn %%xcc, 1b\n"
-" mov %%g5, %0\n"
+" mov %1, %0\n"
" membar #StoreLoad | #StoreStore\n"
- : "=&r" (val)
+ : "=&r" (val), "=&r" (tmp1), "=&r" (tmp2)
: "0" (val), "r" (m)
- : "g5", "g7", "cc", "memory");
+ : "cc", "memory");
return val;
}
-
To unsubscribe from this list: send the line "unsubscribe bk-commits-head" in
the body of a message to [EMAIL PROTECTED]
More majordomo info at http://vger.kernel.org/majordomo-info.html
