Author: mjg
Date: Wed Feb 12 11:12:13 2020
New Revision: 357805
URL: https://svnweb.freebsd.org/changeset/base/357805

Log:
  amd64: store per-cpu allocations subtracted by __pcpu
  
  This eliminates a runtime subtraction from counter_u64_add.
  
  before:
  mov    0x4f00ed(%rip),%rax        # 0xffffffff80c01788 <numfullpathfail4>
  sub    0x808ff6(%rip),%rax        # 0xffffffff80f1a698 <__pcpu>
  addq   $0x1,%gs:(%rax)
  
  after:
  mov    0x4f02fd(%rip),%rax        # 0xffffffff80c01788 <numfullpathfail4>
  addq   $0x1,%gs:(%rax)
  
  Reviewed by:  jeff
  Differential Revision:        https://reviews.freebsd.org/D23570

Modified:
  head/sys/amd64/include/counter.h
  head/sys/amd64/include/pcpu.h

Modified: head/sys/amd64/include/counter.h
==============================================================================
--- head/sys/amd64/include/counter.h    Wed Feb 12 11:11:22 2020        
(r357804)
+++ head/sys/amd64/include/counter.h    Wed Feb 12 11:12:13 2020        
(r357805)
@@ -33,7 +33,7 @@
 
 #include <sys/pcpu.h>
 
-#define        EARLY_COUNTER   &temp_bsp_pcpu.pc_early_dummy_counter
+#define        EARLY_COUNTER   (void *)__offsetof(struct pcpu, 
pc_early_dummy_counter)
 
 #define        counter_enter() do {} while (0)
 #define        counter_exit()  do {} while (0)
@@ -43,6 +43,7 @@ static inline uint64_t
 counter_u64_read_one(counter_u64_t c, int cpu)
 {
 
+       MPASS(c != EARLY_COUNTER);
        return (*zpcpu_get_cpu(c, cpu));
 }
 
@@ -65,6 +66,7 @@ counter_u64_zero_one_cpu(void *arg)
        counter_u64_t c;
 
        c = arg;
+       MPASS(c != EARLY_COUNTER);
        *(zpcpu_get(c)) = 0;
 }
 
@@ -86,7 +88,7 @@ counter_u64_add(counter_u64_t c, int64_t inc)
        KASSERT(IS_BSP() || c != EARLY_COUNTER, ("EARLY_COUNTER used on AP"));
        __asm __volatile("addq\t%1,%%gs:(%0)"
            :
-           : "r" ((char *)c - (char *)&__pcpu[0]), "ri" (inc)
+           : "r" (c), "ri" (inc)
            : "memory", "cc");
 }
 

Modified: head/sys/amd64/include/pcpu.h
==============================================================================
--- head/sys/amd64/include/pcpu.h       Wed Feb 12 11:11:22 2020        
(r357804)
+++ head/sys/amd64/include/pcpu.h       Wed Feb 12 11:12:13 2020        
(r357805)
@@ -240,6 +240,10 @@ _Static_assert(sizeof(struct monitorbuf) == 128, "2x c
 
 #define        IS_BSP()        (PCPU_GET(cpuid) == 0)
 
+#define zpcpu_offset_cpu(cpu)  ((uintptr_t)&__pcpu[0] + UMA_PCPU_ALLOC_SIZE * 
cpu)
+#define zpcpu_base_to_offset(base) (void *)((uintptr_t)(base) - 
(uintptr_t)&__pcpu[0])
+#define zpcpu_offset_to_base(base) (void *)((uintptr_t)(base) + 
(uintptr_t)&__pcpu[0])
+
 #else /* !__GNUCLIKE_ASM || !__GNUCLIKE___TYPEOF */
 
 #error "this file needs to be ported to your compiler"
_______________________________________________
svn-src-head@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-head
To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"

Reply via email to