on modern sparc64s (think fire or sparc enterprise Mx000 boxes),
setting up and tearing down the translation table entries (TTEs)
is very expensive. so expensive that the cost of doing it for disk
io has a noticable impact on compile times.
now that there's a BUS_DMA_64BIT flag, we can use that to decide
to bypass the iommu for devices that set that flag, therefore
avoiding the cost of handling the TTEs.
the following diff adds support for bypass mappings to the iommu
code on sparc64. it's based on a diff from kettenis@ back in 2009.
the main changes are around coping with the differences between
schizo/psycho and fire/oberon.
the differences between the chips are now represented by a iommu_hw
struct. these differences include how to enable the iommu (now via
a function pointer), and masks for bypass addresses.
ive tested this on oberon (on an m4000) and schizo (on a v880).
however, the bypass code isnt working on fire (v245s). to cope with
that for now, the iommu_hw struct lets drivers mask flag bits that
are handled when creating a dmamap. this means fire boards will
ignore BUS_DMA_64BIT until i can figure out whats wrong with them.
i have not tested this on psycho yet. if anyone has such a machine
and is willing to work with me to figure it out, please talk to me.
Index: dev/iommu.c
===================================================================
RCS file: /cvs/src/sys/arch/sparc64/dev/iommu.c,v
retrieving revision 1.74
diff -u -p -r1.74 iommu.c
--- dev/iommu.c 30 Apr 2017 16:45:45 -0000 1.74
+++ dev/iommu.c 8 May 2017 00:45:05 -0000
@@ -100,6 +100,25 @@ void iommu_iomap_clear_pages(struct iomm
void _iommu_dvmamap_sync(bus_dma_tag_t, bus_dma_tag_t, bus_dmamap_t,
bus_addr_t, bus_size_t, int);
+void iommu_hw_enable(struct iommu_state *);
+
+const struct iommu_hw iommu_hw_default = {
+ .ihw_enable = iommu_hw_enable,
+
+ .ihw_dvma_pa = IOTTE_PAMASK,
+
+ .ihw_bypass = 0x3fffUL << 50,
+ .ihw_bypass_nc = 0,
+ .ihw_bypass_ro = 0,
+};
+
+void
+iommu_hw_enable(struct iommu_state *is)
+{
+ IOMMUREG_WRITE(is, iommu_tsb, is->is_ptsb);
+ IOMMUREG_WRITE(is, iommu_cr, IOMMUCR_EN | (is->is_tsbsize << 16));
+}
+
/*
* Initiate an STC entry flush.
*/
@@ -125,7 +144,8 @@ iommu_strbuf_flush(struct strbuf_ctl *sb
* - create a private DVMA map.
*/
void
-iommu_init(char *name, struct iommu_state *is, int tsbsize, u_int32_t iovabase)
+iommu_init(char *name, const struct iommu_hw *ihw, struct iommu_state *is,
+ int tsbsize, u_int32_t iovabase)
{
psize_t size;
vaddr_t va;
@@ -149,13 +169,9 @@ iommu_init(char *name, struct iommu_stat
* be hard-wired, so we read the start and size from the PROM and
* just use those values.
*/
- if (strncmp(name, "pyro", 4) == 0) {
- is->is_cr = IOMMUREG_READ(is, iommu_cr);
- is->is_cr &= ~IOMMUCR_FIRE_BE;
- is->is_cr |= (IOMMUCR_FIRE_SE | IOMMUCR_FIRE_CM_EN |
- IOMMUCR_FIRE_TE);
- } else
- is->is_cr = IOMMUCR_EN;
+
+ is->is_hw = ihw;
+
is->is_tsbsize = tsbsize;
if (iovabase == (u_int32_t)-1) {
is->is_dvmabase = IOTSB_VSTART(is->is_tsbsize);
@@ -237,15 +253,6 @@ iommu_init(char *name, struct iommu_stat
mtx_init(&is->is_mtx, IPL_HIGH);
/*
- * Set the TSB size. The relevant bits were moved to the TSB
- * base register in the PCIe host bridges.
- */
- if (strncmp(name, "pyro", 4) == 0)
- is->is_ptsb |= is->is_tsbsize;
- else
- is->is_cr |= (is->is_tsbsize << 16);
-
- /*
* Now actually start up the IOMMU.
*/
iommu_reset(is);
@@ -262,10 +269,7 @@ iommu_reset(struct iommu_state *is)
{
int i;
- IOMMUREG_WRITE(is, iommu_tsb, is->is_ptsb);
-
- /* Enable IOMMU */
- IOMMUREG_WRITE(is, iommu_cr, is->is_cr);
+ (*is->is_hw->ihw_enable)(is);
for (i = 0; i < 2; ++i) {
struct strbuf_ctl *sb = is->is_sb[i];
@@ -280,7 +284,7 @@ iommu_reset(struct iommu_state *is)
printf(", STC%d enabled", i);
}
- if (is->is_flags & IOMMU_FLUSH_CACHE)
+ if (ISSET(is->is_hw->ihw_flags, IOMMU_HW_FLUSH_CACHE))
IOMMUREG_WRITE(is, iommu_cache_invalidate, -1ULL);
}
@@ -433,7 +437,7 @@ iommu_extract(struct iommu_state *is, bu
if (dva >= is->is_dvmabase && dva <= is->is_dvmaend)
tte = is->is_tsb[IOTSBSLOT(dva, is->is_tsbsize)];
- return (tte & IOTTE_PAMASK);
+ return (tte & is->is_hw->ihw_dvma_pa);
}
/*
@@ -601,8 +605,11 @@ iommu_dvmamap_create(bus_dma_tag_t t, bu
{
int ret;
bus_dmamap_t map;
+ struct iommu_state *is = sb->sb_iommu;
struct iommu_map_state *ims;
+ flags &= ~is->is_hw->ihw_dma_flags;
+
BUS_DMA_FIND_PARENT(t, _dmamap_create);
ret = (*t->_dmamap_create)(t, t0, size, nsegments, maxsegsz, boundary,
flags, &map);
@@ -610,6 +617,12 @@ iommu_dvmamap_create(bus_dma_tag_t t, bu
if (ret)
return (ret);
+ if (flags & BUS_DMA_64BIT) {
+ map->_dm_cookie = is;
+ *dmamap = map;
+ return (0);
+ }
+
ims = iommu_iomap_create(atop(round_page(size)));
if (ims == NULL) {
@@ -641,8 +654,10 @@ iommu_dvmamap_destroy(bus_dma_tag_t t, b
if (map->dm_nsegs)
bus_dmamap_unload(t0, map);
- if (map->_dm_cookie)
- iommu_iomap_destroy(map->_dm_cookie);
+ if (!ISSET(map->_dm_flags, BUS_DMA_64BIT)) {
+ if (map->_dm_cookie)
+ iommu_iomap_destroy(map->_dm_cookie);
+ }
map->_dm_cookie = NULL;
BUS_DMA_FIND_PARENT(t, _dmamap_destroy);
@@ -667,36 +682,36 @@ iommu_dvmamap_load(bus_dma_tag_t t, bus_
u_long dvmaddr, sgstart, sgend;
bus_size_t align, boundary;
struct iommu_state *is;
- struct iommu_map_state *ims = map->_dm_cookie;
+ struct iommu_map_state *ims;
pmap_t pmap;
-#ifdef DIAGNOSTIC
- if (ims == NULL)
- panic("iommu_dvmamap_load: null map state");
-#endif
-#ifdef DEBUG
- if (ims->ims_sb == NULL)
- panic("iommu_dvmamap_load: null sb");
- if (ims->ims_sb->sb_iommu == NULL)
- panic("iommu_dvmamap_load: null iommu");
-#endif /* DEBUG */
- is = ims->ims_sb->sb_iommu;
-
- if (map->dm_nsegs) {
- /*
- * Is it still in use? _bus_dmamap_load should have taken care
- * of this.
- */
-#ifdef DIAGNOSTIC
- panic("iommu_dvmamap_load: map still in use");
-#endif
- bus_dmamap_unload(t0, map);
- }
-
/*
* Make sure that on error condition we return "no valid mappings".
*/
- map->dm_nsegs = 0;
+ KASSERTMSG(map->dm_nsegs == 0, "map still in use");
+
+ if (ISSET(map->_dm_flags, BUS_DMA_64BIT)) {
+ unsigned long bypass;
+ int i;
+
+ is = map->_dm_cookie;
+ bypass = is->is_hw->ihw_bypass;
+
+ /* Bypass translation by the IOMMU. */
+
+ BUS_DMA_FIND_PARENT(t, _dmamap_load);
+ err = (*t->_dmamap_load)(t, t0, map, buf, buflen, p, flags);
+ if (err != 0)
+ return (err);
+
+ for (i = 0; i < map->dm_nsegs; i++)
+ map->dm_segs[i].ds_addr |= bypass;
+
+ return (0);
+ }
+
+ ims = map->_dm_cookie;
+ is = ims->ims_sb->sb_iommu;
if (buflen < 1 || buflen > map->_dm_size) {
DPRINTF(IDB_BUSDMA,
@@ -876,28 +891,31 @@ iommu_dvmamap_load_raw(bus_dma_tag_t t,
bus_size_t boundary, align;
u_long dvmaddr, sgstart, sgend;
struct iommu_state *is;
- struct iommu_map_state *ims = map->_dm_cookie;
+ struct iommu_map_state *ims;
-#ifdef DIAGNOSTIC
- if (ims == NULL)
- panic("iommu_dvmamap_load_raw: null map state");
-#endif
-#ifdef DEBUG
- if (ims->ims_sb == NULL)
- panic("iommu_dvmamap_load_raw: null sb");
- if (ims->ims_sb->sb_iommu == NULL)
- panic("iommu_dvmamap_load_raw: null iommu");
-#endif /* DEBUG */
- is = ims->ims_sb->sb_iommu;
+ KASSERTMSG(map->dm_nsegs == 0, "map stil in use");
- if (map->dm_nsegs) {
- /* Already in use?? */
-#ifdef DIAGNOSTIC
- panic("iommu_dvmamap_load_raw: map still in use");
-#endif
- bus_dmamap_unload(t0, map);
+ if (ISSET(map->_dm_flags, BUS_DMA_64BIT)) {
+ unsigned long bypass;
+
+ is = map->_dm_cookie;
+ bypass = is->is_hw->ihw_bypass;
+
+ /* Bypass translation by the IOMMU. */
+ for (i = 0; i < nsegs; i++) {
+ map->dm_segs[i].ds_addr = bypass | segs[i].ds_addr;
+ map->dm_segs[i].ds_len = segs[i].ds_len;
+ }
+
+ map->dm_nsegs = nsegs;
+ map->dm_mapsize = size;
+
+ return (0);
}
+ ims = map->_dm_cookie;
+ is = ims->ims_sb->sb_iommu;
+
/*
* A boundary presented to bus_dmamem_alloc() takes precedence
* over boundary in the map.
@@ -1088,11 +1106,6 @@ iommu_dvmamap_append_range(bus_dma_tag_t
bus_dma_segment_t *seg = NULL;
int i = map->dm_nsegs;
-#ifdef DEBUG
- if (ims == NULL)
- panic("iommu_dvmamap_append_range: null map state");
-#endif
-
sgstart = iommu_iomap_translate(ims, pa);
sgend = sgstart + length - 1;
@@ -1298,20 +1311,17 @@ void
iommu_dvmamap_unload(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map)
{
struct iommu_state *is;
- struct iommu_map_state *ims = map->_dm_cookie;
+ struct iommu_map_state *ims;
bus_addr_t dvmaddr = map->_dm_dvmastart;
bus_size_t sgsize = map->_dm_dvmasize;
int error;
-#ifdef DEBUG
- if (ims == NULL)
- panic("iommu_dvmamap_unload: null map state");
- if (ims->ims_sb == NULL)
- panic("iommu_dvmamap_unload: null sb");
- if (ims->ims_sb->sb_iommu == NULL)
- panic("iommu_dvmamap_unload: null iommu");
-#endif /* DEBUG */
+ if (ISSET(map->_dm_flags, BUS_DMA_64BIT)) {
+ bus_dmamap_unload(t->_parent, map);
+ return;
+ }
+ ims = map->_dm_cookie;
is = ims->ims_sb->sb_iommu;
/* Flush the iommu */
@@ -1488,7 +1498,7 @@ iommu_dvmamap_print_map(bus_dma_tag_t t,
break;
}
- if (map->_dm_cookie) {
+ if (!ISSET(map->_dm_flags, BUS_DMA_64BIT) && map->_dm_cookie != NULL) {
struct iommu_map_state *ims = map->_dm_cookie;
struct iommu_page_map *ipm = &ims->ims_map;
@@ -1546,19 +1556,19 @@ void
iommu_dvmamap_sync(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map,
bus_addr_t offset, bus_size_t len, int ops)
{
- struct iommu_map_state *ims = map->_dm_cookie;
+ struct iommu_map_state *ims;
-#ifdef DIAGNOSTIC
- if (ims == NULL)
- panic("iommu_dvmamap_sync: null map state");
- if (ims->ims_sb == NULL)
- panic("iommu_dvmamap_sync: null sb");
- if (ims->ims_sb->sb_iommu == NULL)
- panic("iommu_dvmamap_sync: null iommu");
-#endif
if (len == 0)
return;
+ if (map->_dm_flags & BUS_DMA_64BIT) {
+ if (ops & (BUS_DMASYNC_PREWRITE | BUS_DMASYNC_POSTREAD))
+ membar(MemIssue);
+ return;
+ }
+
+ ims = map->_dm_cookie;
+
if (ops & BUS_DMASYNC_PREWRITE)
membar(MemIssue);
@@ -1622,9 +1632,13 @@ iommu_dvmamem_alloc(bus_dma_tag_t t, bus
"bound %llx segp %p flags %d\n", (unsigned long long)size,
(unsigned long long)alignment, (unsigned long long)boundary,
segs, flags));
+
+ if ((flags & BUS_DMA_64BIT) == 0)
+ flags |= BUS_DMA_DVMA;
+
BUS_DMA_FIND_PARENT(t, _dmamem_alloc);
return ((*t->_dmamem_alloc)(t, t0, size, alignment, boundary,
- segs, nsegs, rsegs, flags | BUS_DMA_DVMA));
+ segs, nsegs, rsegs, flags));
}
void
@@ -1763,7 +1777,7 @@ iommu_iomap_load_map(struct iommu_state
/* Flush cache if necessary. */
slot = IOTSBSLOT(e->ipe_va, is->is_tsbsize);
- if (is->is_flags & IOMMU_FLUSH_CACHE &&
+ if (ISSET(is->is_hw->ihw_flags, IOMMU_HW_FLUSH_CACHE) &&
(i == (ipm->ipm_pagecnt - 1) || (slot % 8) == 7))
IOMMUREG_WRITE(is, iommu_cache_flush,
is->is_ptsb + slot * 8);
@@ -1788,7 +1802,7 @@ iommu_iomap_unload_map(struct iommu_stat
/* Flush cache if necessary. */
slot = IOTSBSLOT(e->ipe_va, is->is_tsbsize);
- if (is->is_flags & IOMMU_FLUSH_CACHE &&
+ if (ISSET(is->is_hw->ihw_flags, IOMMU_HW_FLUSH_CACHE) &&
(i == (ipm->ipm_pagecnt - 1) || (slot % 8) == 7))
IOMMUREG_WRITE(is, iommu_cache_flush,
is->is_ptsb + slot * 8);
Index: dev/iommureg.h
===================================================================
RCS file: /cvs/src/sys/arch/sparc64/dev/iommureg.h,v
retrieving revision 1.17
diff -u -p -r1.17 iommureg.h
--- dev/iommureg.h 17 Aug 2012 20:46:50 -0000 1.17
+++ dev/iommureg.h 8 May 2017 00:45:05 -0000
@@ -90,10 +90,10 @@ struct iommu_strbuf {
#define IOMMUCR_DE 0x000000000000000002LL /* Diag enable */
#define IOMMUCR_EN 0x000000000000000001LL /* Enable IOMMU */
-#define IOMMUCR_FIRE_SE 0x000000000000000400LL /* Snoop enable
*/
-#define IOMMUCR_FIRE_CM_EN 0x000000000000000300LL /* Cache mode enable */
-#define IOMMUCR_FIRE_BE 0x000000000000000002LL /* Bypass
enable */
-#define IOMMUCR_FIRE_TE 0x000000000000000001LL /* Translation
enabled */
+#define IOMMUCR_FIRE_SE 0x000000000000000400UL /* Snoop enable
*/
+#define IOMMUCR_FIRE_CM_EN 0x000000000000000300UL /* Cache mode enable */
+#define IOMMUCR_FIRE_BE 0x000000000000000002UL /* Bypass
enable */
+#define IOMMUCR_FIRE_TE 0x000000000000000001UL /* Translation
enabled */
/*
* IOMMU stuff
Index: dev/iommuvar.h
===================================================================
RCS file: /cvs/src/sys/arch/sparc64/dev/iommuvar.h,v
retrieving revision 1.17
diff -u -p -r1.17 iommuvar.h
--- dev/iommuvar.h 4 May 2016 18:26:12 -0000 1.17
+++ dev/iommuvar.h 8 May 2017 00:45:05 -0000
@@ -100,6 +100,22 @@ struct iommu_map_state {
};
#define IOMMU_MAP_STREAM 1
+struct iommu_hw {
+ void (*ihw_enable)(struct iommu_state *);
+
+ unsigned long ihw_dvma_pa;
+
+ unsigned long ihw_bypass;
+ unsigned long ihw_bypass_nc; /* non-cached */
+ unsigned long ihw_bypass_ro; /* relaxed ordering */
+
+ unsigned int ihw_flags;
+#define IOMMU_HW_FLUSH_CACHE (1 << 0)
+ int ihw_dma_flags;
+};
+
+extern const struct iommu_hw iommu_hw_default;
+
/*
* per-IOMMU state
*/
@@ -112,8 +128,7 @@ struct iommu_state {
int64_t is_cr; /* Control register value */
struct mutex is_mtx;
struct extent *is_dvmamap; /* DVMA map for this instance */
- int is_flags;
-#define IOMMU_FLUSH_CACHE 0x00000001
+ const struct iommu_hw *is_hw;
struct strbuf_ctl *is_sb[2]; /* Streaming buffers if any */
@@ -126,7 +141,8 @@ struct iommu_state {
};
/* interfaces for PCI/SBus code */
-void iommu_init(char *, struct iommu_state *, int, u_int32_t);
+void iommu_init(char *, const struct iommu_hw *, struct iommu_state *,
+ int, u_int32_t);
void iommu_reset(struct iommu_state *);
paddr_t iommu_extract(struct iommu_state *, bus_addr_t);
int64_t iommu_lookup_tte(struct iommu_state *, bus_addr_t);
@@ -146,6 +162,7 @@ int iommu_dvmamem_alloc(bus_dma_tag_t, b
bus_size_t, bus_size_t, bus_dma_segment_t *, int, int *, int);
void iommu_dvmamem_free(bus_dma_tag_t, bus_dma_tag_t, bus_dma_segment_t *,
int);
+
#define IOMMUREG_READ(is, reg) \
bus_space_read_8((is)->is_bustag, \
Index: dev/psycho.c
===================================================================
RCS file: /cvs/src/sys/arch/sparc64/dev/psycho.c,v
retrieving revision 1.74
diff -u -p -r1.74 psycho.c
--- dev/psycho.c 23 Aug 2016 03:28:01 -0000 1.74
+++ dev/psycho.c 8 May 2017 00:45:05 -0000
@@ -902,7 +902,7 @@ psycho_iommu_init(struct psycho_softc *s
panic("couldn't malloc iommu name");
snprintf(name, 32, "%s dvma", sc->sc_dev.dv_xname);
- iommu_init(name, is, tsbsize, iobase);
+ iommu_init(name, &iommu_hw_default, is, tsbsize, iobase);
}
/*
Index: dev/pyro.c
===================================================================
RCS file: /cvs/src/sys/arch/sparc64/dev/pyro.c,v
retrieving revision 1.30
diff -u -p -r1.30 pyro.c
--- dev/pyro.c 20 Dec 2016 13:40:50 -0000 1.30
+++ dev/pyro.c 8 May 2017 00:45:05 -0000
@@ -131,6 +131,32 @@ int pyro_msi_eq_intr(void *);
int pyro_dmamap_create(bus_dma_tag_t, bus_dma_tag_t, bus_size_t, int,
bus_size_t, bus_size_t, int, bus_dmamap_t *);
+void pyro_iommu_enable(struct iommu_state *);
+
+const struct iommu_hw iommu_hw_pyro = {
+ .ihw_enable = pyro_iommu_enable,
+
+ .ihw_dvma_pa = 0x7ffffffffffUL,
+
+ .ihw_bypass = 0xfffc000000000000UL,
+ .ihw_bypass_nc = 1UL << 43,
+ .ihw_bypass_ro = 0,
+
+ .ihw_dma_flags = BUS_DMA_64BIT,
+};
+
+const struct iommu_hw iommu_hw_oberon = {
+ .ihw_enable = pyro_iommu_enable,
+
+ .ihw_dvma_pa = 0x7fffffffffUL,
+
+ .ihw_bypass = 0x7ffc000000000000UL,
+ .ihw_bypass_nc = 1UL << 47,
+ .ihw_bypass_ro = 0x8000000000000000UL,
+
+ .ihw_flags = IOMMU_HW_FLUSH_CACHE,
+};
+
#ifdef DDB
void pyro_xir(void *, int);
#endif
@@ -266,6 +292,7 @@ pyro_init_iommu(struct pyro_softc *sc, s
int tsbsize = 7;
u_int32_t iobase = -1;
char *name;
+ const struct iommu_hw *ihw = &iommu_hw_pyro;
is->is_bustag = sc->sc_bust;
@@ -282,11 +309,23 @@ pyro_init_iommu(struct pyro_softc *sc, s
panic("couldn't malloc iommu name");
snprintf(name, 32, "%s dvma", sc->sc_dv.dv_xname);
- /* On Oberon, we need to flush the cache. */
if (sc->sc_oberon)
- is->is_flags |= IOMMU_FLUSH_CACHE;
+ ihw = &iommu_hw_oberon;
+
+ iommu_init(name, ihw, is, tsbsize, iobase);
+}
+
+void
+pyro_iommu_enable(struct iommu_state *is)
+{
+ unsigned long cr;
+
+ cr = IOMMUREG_READ(is, iommu_cr);
+ cr |= IOMMUCR_FIRE_BE | IOMMUCR_FIRE_SE | IOMMUCR_FIRE_CM_EN |
+ IOMMUCR_FIRE_TE;
- iommu_init(name, is, tsbsize, iobase);
+ IOMMUREG_WRITE(is, iommu_tsb, is->is_ptsb | is->is_tsbsize);
+ IOMMUREG_WRITE(is, iommu_cr, cr);
}
void
Index: dev/sbus.c
===================================================================
RCS file: /cvs/src/sys/arch/sparc64/dev/sbus.c,v
retrieving revision 1.44
diff -u -p -r1.44 sbus.c
--- dev/sbus.c 19 Sep 2015 21:07:04 -0000 1.44
+++ dev/sbus.c 8 May 2017 00:45:05 -0000
@@ -349,7 +349,7 @@ sbus_mb_attach(struct device *parent, st
snprintf(name, 32, "%s dvma", sc->sc_dev.dv_xname);
printf("%s: ", sc->sc_dev.dv_xname);
- iommu_init(name, &sc->sc_is, 0, -1);
+ iommu_init(name, &iommu_hw_default, &sc->sc_is, 0, -1);
/* Initialize Starfire PC interrupt translation. */
if (OF_getprop(findroot(), "name", buf, sizeof(buf)) > 0 &&
Index: dev/schizo.c
===================================================================
RCS file: /cvs/src/sys/arch/sparc64/dev/schizo.c,v
retrieving revision 1.67
diff -u -p -r1.67 schizo.c
--- dev/schizo.c 23 Aug 2016 03:28:01 -0000 1.67
+++ dev/schizo.c 8 May 2017 00:45:05 -0000
@@ -451,7 +451,7 @@ schizo_init_iommu(struct schizo_softc *s
"using iobase=0x%x, tsbsize=%d\n", iobase, tsbsize));
}
- iommu_init(name, is, tsbsize, iobase);
+ iommu_init(name, &iommu_hw_default, is, tsbsize, iobase);
}
int
