[PATCH net-next] cxgb4: Allocate Tx queues dynamically
From: Hariprasad Shenai
Allocate resources dynamically for Upper layer driver's (ULD) like
cxgbit, iw_cxgb4, cxgb4i and chcr. The resources allocated include Tx
queues which are allocated when ULD register with cxgb4 driver and freed
while un-registering. The Tx queues which are shared by ULD shall be
allocated by first registering driver and un-allocated by last
unregistering driver.
Signed-off-by: Atul Gupta
---
drivers/crypto/chelsio/chcr_algo.c | 16 +--
drivers/crypto/chelsio/chcr_core.c | 3 +-
drivers/infiniband/hw/cxgb4/device.c | 1 +
drivers/net/ethernet/chelsio/cxgb4/cxgb4.h | 19 +++-
drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c | 12 --
drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c| 64 +++
drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c | 114 +++
drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h | 17 +++
drivers/net/ethernet/chelsio/cxgb4/sge.c | 121 +++--
drivers/scsi/cxgbi/cxgb4i/cxgb4i.c | 1 +
drivers/target/iscsi/cxgbit/cxgbit_main.c | 1 +
11 files changed, 287 insertions(+), 82 deletions(-)
diff --git a/drivers/crypto/chelsio/chcr_algo.c
b/drivers/crypto/chelsio/chcr_algo.c
index e4ddb921d7b3..56b153805462 100644
--- a/drivers/crypto/chelsio/chcr_algo.c
+++ b/drivers/crypto/chelsio/chcr_algo.c
@@ -592,16 +592,18 @@ static int chcr_aes_cbc_setkey(struct crypto_ablkcipher
*tfm, const u8 *key,
static int cxgb4_is_crypto_q_full(struct net_device *dev, unsigned int idx)
{
- int ret = 0;
- struct sge_ofld_txq *q;
struct adapter *adap = netdev2adap(dev);
+ struct sge_uld_txq_info *txq_info =
+ adap->sge.uld_txq_info[CXGB4_TX_CRYPTO];
+ struct sge_uld_txq *txq;
+ int ret = 0;
local_bh_disable();
- q = &adap->sge.ofldtxq[idx];
- spin_lock(&q->sendq.lock);
- if (q->full)
+ txq = &txq_info->uldtxq[idx];
+ spin_lock(&txq->sendq.lock);
+ if (txq->full)
ret = -1;
- spin_unlock(&q->sendq.lock);
+ spin_unlock(&txq->sendq.lock);
local_bh_enable();
return ret;
}
@@ -674,11 +676,11 @@ static int chcr_device_init(struct chcr_context *ctx)
}
u_ctx = ULD_CTX(ctx);
rxq_perchan = u_ctx->lldi.nrxq / u_ctx->lldi.nchan;
- ctx->dev->tx_channel_id = 0;
rxq_idx = ctx->dev->tx_channel_id * rxq_perchan;
rxq_idx += id % rxq_perchan;
spin_lock(&ctx->dev->lock_chcr_dev);
ctx->tx_channel_id = rxq_idx;
+ ctx->dev->tx_channel_id = !ctx->dev->tx_channel_id;
spin_unlock(&ctx->dev->lock_chcr_dev);
}
out:
diff --git a/drivers/crypto/chelsio/chcr_core.c
b/drivers/crypto/chelsio/chcr_core.c
index fb5f9bbfa09c..4d7f6700fd7e 100644
--- a/drivers/crypto/chelsio/chcr_core.c
+++ b/drivers/crypto/chelsio/chcr_core.c
@@ -42,6 +42,7 @@ static chcr_handler_func work_handlers[NUM_CPL_CMDS] = {
static struct cxgb4_uld_info chcr_uld_info = {
.name = DRV_MODULE_NAME,
.nrxq = MAX_ULD_QSETS,
+ .ntxq = MAX_ULD_QSETS,
.rxq_size = 1024,
.add = chcr_uld_add,
.state_change = chcr_uld_state_change,
@@ -126,7 +127,7 @@ static int cpl_fw6_pld_handler(struct chcr_dev *dev,
int chcr_send_wr(struct sk_buff *skb)
{
- return cxgb4_ofld_send(skb->dev, skb);
+ return cxgb4_crypto_send(skb->dev, skb);
}
static void *chcr_uld_add(const struct cxgb4_lld_info *lld)
diff --git a/drivers/infiniband/hw/cxgb4/device.c
b/drivers/infiniband/hw/cxgb4/device.c
index 93e3d270a98a..4e5baf4fe15e 100644
--- a/drivers/infiniband/hw/cxgb4/device.c
+++ b/drivers/infiniband/hw/cxgb4/device.c
@@ -1481,6 +1481,7 @@ static int c4iw_uld_control(void *handle, enum
cxgb4_control control, ...)
static struct cxgb4_uld_info c4iw_uld_info = {
.name = DRV_NAME,
.nrxq = MAX_ULD_QSETS,
+ .ntxq = MAX_ULD_QSETS,
.rxq_size = 511,
.ciq = true,
.lro = false,
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
index 2125903043fb..0bce1bf9ca0f 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
@@ -635,6 +635,7 @@ struct tx_sw_desc;
struct sge_txq {
unsigned int in_use; /* # of in-use Tx descriptors */
+ unsigned int q_type; /* Q type Eth/Ctrl/Ofld */
unsigned int size; /* # of descriptors */
unsigned int cidx; /* SW consumer index */
unsigned int pidx; /* producer index */
@@ -665,7 +666,7 @@ struct sge_eth_txq {/* state for an SGE
Ethernet Tx queue */
unsigned long mapping_err; /* # of I/
[PATCH net-next 1/2] cxgb4: Add support for Inline IPSec Tx
Added Tx routine for ULD
- define interface for ULD Tx.
Export routines used for Tx data
- Routines common for data transmit are used by cxgb4 and chcr
drivers.
- EXPORT routines enable transmit from chcr driver.
Signed-off-by: Atul Gupta
Signed-off-by: Ganesh Goudar
---
drivers/net/ethernet/chelsio/cxgb4/cxgb4.h | 23 +
drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c | 2 +
drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c| 2 +-
drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c | 1 +
drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h | 3 +
drivers/net/ethernet/chelsio/cxgb4/sge.c | 101 ++---
6 files changed, 80 insertions(+), 52 deletions(-)
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
index 92a0b02..ee3832a 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
@@ -58,6 +58,13 @@
extern struct list_head adapter_list;
extern struct mutex uld_mutex;
+/* Suspend an Ethernet Tx queue with fewer available descriptors than this.
+ * This is the same as calc_tx_descs() for a TSO packet with
+ * nr_frags == MAX_SKB_FRAGS.
+ */
+#define ETHTXQ_STOP_THRES \
+ (1 + DIV_ROUND_UP((3 * MAX_SKB_FRAGS) / 2 + (MAX_SKB_FRAGS & 1), 8))
+
enum {
MAX_NPORTS = 4, /* max # of ports */
SERNUM_LEN = 24,/* Serial # length */
@@ -555,6 +562,7 @@ enum { /* adapter flags */
enum {
ULP_CRYPTO_LOOKASIDE = 1 << 0,
+ ULP_CRYPTO_IPSEC_INLINE = 1 << 1,
};
struct rx_sw_desc;
@@ -957,6 +965,11 @@ enum {
SCHED_CLASS_RATEMODE_ABS = 1, /* Kb/s */
};
+struct tx_sw_desc {/* SW state per Tx descriptor */
+ struct sk_buff *skb;
+ struct ulptx_sgl *sgl;
+};
+
/* Support for "sched_queue" command to allow one or more NIC TX Queues
* to be bound to a TX Scheduling Class.
*/
@@ -1662,4 +1675,14 @@ void t4_tp_mib_read(struct adapter *adap, u32 *buff, u32
nregs,
void free_tx_desc(struct adapter *adap, struct sge_txq *q,
unsigned int n, bool unmap);
void free_txq(struct adapter *adap, struct sge_txq *q);
+inline void cxgb4_reclaim_completed_tx(struct adapter *adap,
+ struct sge_txq *q, bool unmap);
+int cxgb4_map_skb(struct device *dev, const struct sk_buff *skb,
+ dma_addr_t *addr);
+void cxgb4_inline_tx_skb(const struct sk_buff *skb, const struct sge_txq *q,
+void *pos);
+void cxgb4_write_sgl(const struct sk_buff *skb, struct sge_txq *q,
+struct ulptx_sgl *sgl, u64 *end, unsigned int start,
+const dma_addr_t *addr);
+inline void cxgb4_ring_tx_db(struct adapter *adap, struct sge_txq *q, int n);
#endif /* __CXGB4_H__ */
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
index 917663b..cf47183 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
@@ -3096,6 +3096,8 @@ static int chcr_show(struct seq_file *seq, void *v)
atomic_read(&adap->chcr_stats.error));
seq_printf(seq, "Fallback: %10u \n",
atomic_read(&adap->chcr_stats.fallback));
+ seq_printf(seq, "IPSec PDU: %10u\n",
+ atomic_read(&adap->chcr_stats.ipsec_cnt));
return 0;
}
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
index e16078d..538a8a7 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
@@ -4079,7 +4079,7 @@ static int adap_init0(struct adapter *adap)
} else {
adap->vres.ncrypto_fc = val[0];
}
- adap->params.crypto |= ULP_CRYPTO_LOOKASIDE;
+ adap->params.crypto = ntohs(caps_cmd.cryptocaps);
adap->num_uld += 1;
}
#undef FW_PARAM_PFVF
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c
b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c
index 71a315b..6b5fea4 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c
@@ -637,6 +637,7 @@ static void uld_init(struct adapter *adap, struct
cxgb4_lld_info *lld)
lld->nchan = adap->params.nports;
lld->nports = adap->params.nports;
lld->wr_cred = adap->params.ofldq_wr_cred;
+ lld->crypto = adap->params.crypto;
lld->iscsi_iolen = MAXRXDATA_G(t4_read_reg(adap, TP_PARA_REG2_A));
lld->iscsi_tagmask = t4_read_reg(adap, ULP_RX_ISCSI_TAGMASK_A);
lld->iscsi_pgsz_order = t4_read_reg(adap, ULP_RX_ISCSI_PSZ_A);
diff --git a/drivers/net/ethernet/chelsio
[PATCH net-next 2/2] chcr: Add support for Inline IPSec
register xfrmdev_ops callbacks, Send IPsec tunneled data
to HW for inline processing.
The driver use hardware crypto accelerator to encrypt and
generate ICV for the transmitted packet in Inline mode.
Signed-off-by: Atul Gupta
Signed-off-by: Harsh Jain
Signed-off-by: Ganesh Goudar
---
drivers/crypto/chelsio/Makefile | 2 +-
drivers/crypto/chelsio/chcr_algo.c | 45 ++-
drivers/crypto/chelsio/chcr_algo.h | 17 -
drivers/crypto/chelsio/chcr_core.c | 8 +
drivers/crypto/chelsio/chcr_core.h | 39 +++
drivers/crypto/chelsio/chcr_crypto.h | 17 +-
drivers/crypto/chelsio/chcr_ipsec.c | 659 +++
7 files changed, 740 insertions(+), 47 deletions(-)
create mode 100644 drivers/crypto/chelsio/chcr_ipsec.c
diff --git a/drivers/crypto/chelsio/Makefile b/drivers/crypto/chelsio/Makefile
index bebdf06..9056420 100644
--- a/drivers/crypto/chelsio/Makefile
+++ b/drivers/crypto/chelsio/Makefile
@@ -1,4 +1,4 @@
ccflags-y := -Idrivers/net/ethernet/chelsio/cxgb4
obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chcr.o
-chcr-objs := chcr_core.o chcr_algo.o
+chcr-objs := chcr_core.o chcr_algo.o chcr_ipsec.o
diff --git a/drivers/crypto/chelsio/chcr_algo.c
b/drivers/crypto/chelsio/chcr_algo.c
index 0e81607..060f747 100644
--- a/drivers/crypto/chelsio/chcr_algo.c
+++ b/drivers/crypto/chelsio/chcr_algo.c
@@ -70,6 +70,29 @@
#include "chcr_algo.h"
#include "chcr_crypto.h"
+unsigned int sgl_ent_len[] = {0, 0, 16, 24, 40,
+ 48, 64, 72, 88,
+ 96, 112, 120, 136,
+ 144, 160, 168, 184, 192};
+
+unsigned int dsgl_ent_len[] = {0, 32, 32, 48, 48, 64, 64, 80, 80,
+ 112, 112, 128, 128, 144, 144, 160, 160,
+ 192, 192, 208, 208, 224, 224, 240, 240,
+ 272, 272, 288, 288, 304, 304, 320, 320};
+
+static u32 round_constant[11] = {
+ 0x0100, 0x0200, 0x0400, 0x0800,
+ 0x1000, 0x2000, 0x4000, 0x8000,
+ 0x1B00, 0x3600, 0x6C00
+};
+
+static int is_newsg(struct scatterlist *sgl, unsigned int *newents);
+static struct scatterlist *alloc_new_sg(struct scatterlist *sgl,
+ unsigned int nents);
+static inline void free_new_sg(struct scatterlist *sgl);
+static int chcr_handle_cipher_resp(struct ablkcipher_request *req,
+ unsigned char *input, int err);
+
static inline struct chcr_aead_ctx *AEAD_CTX(struct chcr_context *ctx)
{
return ctx->crypto_ctx->aeadctx;
@@ -105,18 +128,6 @@ static inline int is_ofld_imm(const struct sk_buff *skb)
return (skb->len <= CRYPTO_MAX_IMM_TX_PKT_LEN);
}
-/*
- * sgl_len - calculates the size of an SGL of the given capacity
- * @n: the number of SGL entries
- * Calculates the number of flits needed for a scatter/gather list that
- * can hold the given number of entries.
- */
-static inline unsigned int sgl_len(unsigned int n)
-{
- n--;
- return (3 * n) / 2 + (n & 1) + 2;
-}
-
static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err)
{
u8 temp[SHA512_DIGEST_SIZE];
@@ -432,7 +443,7 @@ static inline int map_writesg_phys_cpl(struct device *dev,
return 0;
}
-static inline int get_aead_subtype(struct crypto_aead *aead)
+inline int get_aead_subtype(struct crypto_aead *aead)
{
struct aead_alg *alg = crypto_aead_alg(aead);
struct chcr_alg_template *chcr_crypto_alg =
@@ -3134,10 +3145,10 @@ static int chcr_aead_decrypt(struct aead_request *req)
}
}
-static int chcr_aead_op(struct aead_request *req,
- unsigned short op_type,
- int size,
- create_wr_t create_wr_fn)
+int chcr_aead_op(struct aead_request *req,
+unsigned short op_type,
+int size,
+create_wr_t create_wr_fn)
{
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct chcr_context *ctx = crypto_aead_ctx(tfm);
diff --git a/drivers/crypto/chelsio/chcr_algo.h
b/drivers/crypto/chelsio/chcr_algo.h
index 583008d..a5f5b19 100644
--- a/drivers/crypto/chelsio/chcr_algo.h
+++ b/drivers/crypto/chelsio/chcr_algo.h
@@ -217,7 +217,6 @@
ULP_TX_SC_MORE_V((immdatalen) ? 0 : 1))
#define MAX_NK 8
-#define CRYPTO_MAX_IMM_TX_PKT_LEN 256
#define MAX_WR_SIZE512
#define ROUND_16(bytes)((bytes) & 0xFFF0)
#define MAX_DSGL_ENT 32
@@ -230,16 +229,6 @@
#define SPACE_LEFT(len) \
((MAX_WR_SIZE - WR_MIN_LEN - (len)))
-unsigned int sgl_ent_len[] = {0, 0, 16, 24, 40,
- 48, 64, 72, 88,
- 96, 112, 120, 136,
- 144, 160, 168, 184,
- 192};
-unsigned int
[PATCH v2 1/2] cxgb4: Add support for Inline IPSec Tx
Added Tx routine for ULD
- define interface for ULD Tx.
Export routines used for Tx data
- Routines common for data transmit are used by cxgb4 and chcr
drivers.
- EXPORT routines enable transmit from chcr driver.
Signed-off-by: Atul Gupta
Signed-off-by: Harsh Jain
Signed-off-by: Ganesh Goudar
---
V2: Fixed the build warnings and created patch against cryptodev
to avoid possible merge conflicts
---
drivers/net/ethernet/chelsio/cxgb4/cxgb4.h | 23 +
drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c | 2 +
drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c| 2 +-
drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c | 1 +
drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h | 3 +
drivers/net/ethernet/chelsio/cxgb4/sge.c | 102 ++---
6 files changed, 81 insertions(+), 52 deletions(-)
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
index ea72d2d..0aa681e 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
@@ -58,6 +58,13 @@
extern struct list_head adapter_list;
extern struct mutex uld_mutex;
+/* Suspend an Ethernet Tx queue with fewer available descriptors than this.
+ * This is the same as calc_tx_descs() for a TSO packet with
+ * nr_frags == MAX_SKB_FRAGS.
+ */
+#define ETHTXQ_STOP_THRES \
+ (1 + DIV_ROUND_UP((3 * MAX_SKB_FRAGS) / 2 + (MAX_SKB_FRAGS & 1), 8))
+
enum {
MAX_NPORTS = 4, /* max # of ports */
SERNUM_LEN = 24,/* Serial # length */
@@ -553,6 +560,7 @@ enum { /* adapter flags */
enum {
ULP_CRYPTO_LOOKASIDE = 1 << 0,
+ ULP_CRYPTO_IPSEC_INLINE = 1 << 1,
};
struct rx_sw_desc;
@@ -947,6 +955,11 @@ enum {
SCHED_CLASS_RATEMODE_ABS = 1, /* Kb/s */
};
+struct tx_sw_desc {/* SW state per Tx descriptor */
+ struct sk_buff *skb;
+ struct ulptx_sgl *sgl;
+};
+
/* Support for "sched_queue" command to allow one or more NIC TX Queues
* to be bound to a TX Scheduling Class.
*/
@@ -1627,4 +1640,14 @@ int t4_set_vf_mac_acl(struct adapter *adapter, unsigned
int vf,
void free_tx_desc(struct adapter *adap, struct sge_txq *q,
unsigned int n, bool unmap);
void free_txq(struct adapter *adap, struct sge_txq *q);
+void cxgb4_reclaim_completed_tx(struct adapter *adap,
+ struct sge_txq *q, bool unmap);
+int cxgb4_map_skb(struct device *dev, const struct sk_buff *skb,
+ dma_addr_t *addr);
+void cxgb4_inline_tx_skb(const struct sk_buff *skb, const struct sge_txq *q,
+void *pos);
+void cxgb4_write_sgl(const struct sk_buff *skb, struct sge_txq *q,
+struct ulptx_sgl *sgl, u64 *end, unsigned int start,
+const dma_addr_t *addr);
+void cxgb4_ring_tx_db(struct adapter *adap, struct sge_txq *q, int n);
#endif /* __CXGB4_H__ */
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
index 76540b0..b45bea9 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
@@ -3095,6 +3095,8 @@ static int chcr_show(struct seq_file *seq, void *v)
atomic_read(&adap->chcr_stats.error));
seq_printf(seq, "Fallback: %10u \n",
atomic_read(&adap->chcr_stats.fallback));
+ seq_printf(seq, "IPSec PDU: %10u\n",
+ atomic_read(&adap->chcr_stats.ipsec_cnt));
return 0;
}
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
index 92d9d79..2243e31 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
@@ -4005,7 +4005,7 @@ static int adap_init0(struct adapter *adap)
} else {
adap->vres.ncrypto_fc = val[0];
}
- adap->params.crypto |= ULP_CRYPTO_LOOKASIDE;
+ adap->params.crypto = ntohs(caps_cmd.cryptocaps);
adap->num_uld += 1;
}
#undef FW_PARAM_PFVF
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c
b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c
index 71a315b..6b5fea4 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c
@@ -637,6 +637,7 @@ static void uld_init(struct adapter *adap, struct
cxgb4_lld_info *lld)
lld->nchan = adap->params.nports;
lld->nports = adap->params.nports;
lld->wr_cred = adap->params.ofldq_wr_cred;
+ lld->crypto = adap->params.crypto;
lld->iscsi_iolen = MAXRXDATA_G(t4_read_reg(adap, TP_PARA_REG2_A));
lld->iscsi_tagmask = t4_read_reg(adap, ULP_RX_ISCSI_TAGMASK
[PATCH v2 2/2] chcr: Add support for Inline IPSec
register xfrmdev_ops callbacks, Send IPsec tunneled data
to HW for inline processing.
The driver use hardware crypto accelerator to encrypt and
generate ICV for the transmitted packet in Inline mode.
Signed-off-by: Atul Gupta
Signed-off-by: Harsh Jain
Signed-off-by: Ganesh Goudar
---
V2: Fixed the build warnings and created patch against cryptodev
to avoid possible merge conflicts
---
drivers/crypto/chelsio/Kconfig | 10 +
drivers/crypto/chelsio/Makefile | 1 +
drivers/crypto/chelsio/chcr_algo.c | 221 ++--
drivers/crypto/chelsio/chcr_algo.h | 15 -
drivers/crypto/chelsio/chcr_core.c | 8 +
drivers/crypto/chelsio/chcr_core.h | 38 ++
drivers/crypto/chelsio/chcr_crypto.h | 69 ++--
drivers/crypto/chelsio/chcr_ipsec.c | 659 +++
8 files changed, 853 insertions(+), 168 deletions(-)
create mode 100644 drivers/crypto/chelsio/chcr_ipsec.c
diff --git a/drivers/crypto/chelsio/Kconfig b/drivers/crypto/chelsio/Kconfig
index 3e104f5..51932c7 100644
--- a/drivers/crypto/chelsio/Kconfig
+++ b/drivers/crypto/chelsio/Kconfig
@@ -18,3 +18,13 @@ config CRYPTO_DEV_CHELSIO
To compile this driver as a module, choose M here: the module
will be called chcr.
+
+config CHELSIO_IPSEC_INLINE
+bool "Chelsio IPSec XFRM Tx crypto offload"
+depends on CHELSIO_T4
+ depends on CRYPTO_DEV_CHELSIO
+depends on XFRM_OFFLOAD
+depends on INET_ESP_OFFLOAD || INET6_ESP_OFFLOAD
+default n
+---help---
+ Enable support for IPSec Tx Inline.
diff --git a/drivers/crypto/chelsio/Makefile b/drivers/crypto/chelsio/Makefile
index bebdf06..eaecaf1 100644
--- a/drivers/crypto/chelsio/Makefile
+++ b/drivers/crypto/chelsio/Makefile
@@ -2,3 +2,4 @@ ccflags-y := -Idrivers/net/ethernet/chelsio/cxgb4
obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chcr.o
chcr-objs := chcr_core.o chcr_algo.o
+chcr-$(CONFIG_CHELSIO_IPSEC_INLINE) += chcr_ipsec.o
diff --git a/drivers/crypto/chelsio/chcr_algo.c
b/drivers/crypto/chelsio/chcr_algo.c
index 4eed717..b623368 100644
--- a/drivers/crypto/chelsio/chcr_algo.c
+++ b/drivers/crypto/chelsio/chcr_algo.c
@@ -73,6 +73,25 @@
#define IV AES_BLOCK_SIZE
+unsigned int sgl_ent_len[] = {0, 0, 16, 24, 40, 48, 64, 72, 88,
+ 96, 112, 120, 136, 144, 160, 168, 184,
+ 192, 208, 216, 232, 240, 256, 264, 280,
+ 288, 304, 312, 328, 336, 352, 360, 376};
+
+unsigned int dsgl_ent_len[] = {0, 32, 32, 48, 48, 64, 64, 80, 80,
+ 112, 112, 128, 128, 144, 144, 160, 160,
+ 192, 192, 208, 208, 224, 224, 240, 240,
+ 272, 272, 288, 288, 304, 304, 320, 320};
+
+static u32 round_constant[11] = {
+ 0x0100, 0x0200, 0x0400, 0x0800,
+ 0x1000, 0x2000, 0x4000, 0x8000,
+ 0x1B00, 0x3600, 0x6C00
+};
+
+static int chcr_handle_cipher_resp(struct ablkcipher_request *req,
+ unsigned char *input, int err);
+
static inline struct chcr_aead_ctx *AEAD_CTX(struct chcr_context *ctx)
{
return ctx->crypto_ctx->aeadctx;
@@ -108,18 +127,6 @@ static inline int is_ofld_imm(const struct sk_buff *skb)
return (skb->len <= SGE_MAX_WR_LEN);
}
-/*
- * sgl_len - calculates the size of an SGL of the given capacity
- * @n: the number of SGL entries
- * Calculates the number of flits needed for a scatter/gather list that
- * can hold the given number of entries.
- */
-static inline unsigned int sgl_len(unsigned int n)
-{
- n--;
- return (3 * n) / 2 + (n & 1) + 2;
-}
-
static int sg_nents_xlen(struct scatterlist *sg, unsigned int reqlen,
unsigned int entlen,
unsigned int skip)
@@ -183,30 +190,17 @@ static inline void chcr_handle_ahash_resp(struct
ahash_request *req,
}
out:
req->base.complete(&req->base, err);
+}
- }
-
-static inline void chcr_handle_aead_resp(struct aead_request *req,
-unsigned char *input,
-int err)
+static inline int get_aead_subtype(struct crypto_aead *aead)
{
- struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
- struct crypto_aead *tfm = crypto_aead_reqtfm(req);
- struct uld_ctx *u_ctx = ULD_CTX(a_ctx(tfm));
-
-
- chcr_aead_dma_unmap(&u_ctx->lldi.pdev->dev, req, reqctx->op);
- if (reqctx->b0_dma)
- dma_unmap_single(&u_ctx->lldi.pdev->dev, reqctx->b0_dma,
-reqctx->b0_len, DMA_BIDIRECTIONAL);
- if (reqctx->verify == VERIFY_SW) {
- chcr_verify_tag(req, input, &err);
- reqctx->verify = VERIFY_HW;
+ struct aead_alg *alg = crypto_aead_alg(aead);
+
[crypto v3 1/2] cxgb4: Add support for Inline IPSec Tx
Added Tx routine for ULD
- define interface for ULD Tx.
Export routines used for Tx data
- Routines common for data transmit are used by cxgb4 and chcr
drivers.
- EXPORT routines enable transmit from chcr driver.
Signed-off-by: Atul Gupta
Signed-off-by: Ganesh Goudar
---
V2: Fixed the build warnings and created patch against cryptodev
to avoid possible merge conflicts
V3: Fixed a build warning and added flag for data packet
---
drivers/net/ethernet/chelsio/cxgb4/cxgb4.h | 23 +
drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c | 2 +
drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c| 2 +-
drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c | 1 +
drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h | 3 +
drivers/net/ethernet/chelsio/cxgb4/sge.c | 102 ++---
drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h | 7 ++
7 files changed, 88 insertions(+), 52 deletions(-)
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
index ea72d2d..0aa681e 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
@@ -58,6 +58,13 @@
extern struct list_head adapter_list;
extern struct mutex uld_mutex;
+/* Suspend an Ethernet Tx queue with fewer available descriptors than this.
+ * This is the same as calc_tx_descs() for a TSO packet with
+ * nr_frags == MAX_SKB_FRAGS.
+ */
+#define ETHTXQ_STOP_THRES \
+ (1 + DIV_ROUND_UP((3 * MAX_SKB_FRAGS) / 2 + (MAX_SKB_FRAGS & 1), 8))
+
enum {
MAX_NPORTS = 4, /* max # of ports */
SERNUM_LEN = 24,/* Serial # length */
@@ -553,6 +560,7 @@ enum { /* adapter flags */
enum {
ULP_CRYPTO_LOOKASIDE = 1 << 0,
+ ULP_CRYPTO_IPSEC_INLINE = 1 << 1,
};
struct rx_sw_desc;
@@ -947,6 +955,11 @@ enum {
SCHED_CLASS_RATEMODE_ABS = 1, /* Kb/s */
};
+struct tx_sw_desc {/* SW state per Tx descriptor */
+ struct sk_buff *skb;
+ struct ulptx_sgl *sgl;
+};
+
/* Support for "sched_queue" command to allow one or more NIC TX Queues
* to be bound to a TX Scheduling Class.
*/
@@ -1627,4 +1640,14 @@ int t4_set_vf_mac_acl(struct adapter *adapter, unsigned
int vf,
void free_tx_desc(struct adapter *adap, struct sge_txq *q,
unsigned int n, bool unmap);
void free_txq(struct adapter *adap, struct sge_txq *q);
+void cxgb4_reclaim_completed_tx(struct adapter *adap,
+ struct sge_txq *q, bool unmap);
+int cxgb4_map_skb(struct device *dev, const struct sk_buff *skb,
+ dma_addr_t *addr);
+void cxgb4_inline_tx_skb(const struct sk_buff *skb, const struct sge_txq *q,
+void *pos);
+void cxgb4_write_sgl(const struct sk_buff *skb, struct sge_txq *q,
+struct ulptx_sgl *sgl, u64 *end, unsigned int start,
+const dma_addr_t *addr);
+void cxgb4_ring_tx_db(struct adapter *adap, struct sge_txq *q, int n);
#endif /* __CXGB4_H__ */
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
index 76540b0..b45bea9 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
@@ -3095,6 +3095,8 @@ static int chcr_show(struct seq_file *seq, void *v)
atomic_read(&adap->chcr_stats.error));
seq_printf(seq, "Fallback: %10u \n",
atomic_read(&adap->chcr_stats.fallback));
+ seq_printf(seq, "IPSec PDU: %10u\n",
+ atomic_read(&adap->chcr_stats.ipsec_cnt));
return 0;
}
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
index 92d9d79..2243e31 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
@@ -4005,7 +4005,7 @@ static int adap_init0(struct adapter *adap)
} else {
adap->vres.ncrypto_fc = val[0];
}
- adap->params.crypto |= ULP_CRYPTO_LOOKASIDE;
+ adap->params.crypto = ntohs(caps_cmd.cryptocaps);
adap->num_uld += 1;
}
#undef FW_PARAM_PFVF
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c
b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c
index 71a315b..6b5fea4 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c
@@ -637,6 +637,7 @@ static void uld_init(struct adapter *adap, struct
cxgb4_lld_info *lld)
lld->nchan = adap->params.nports;
lld->nports = adap->params.nports;
lld->wr_cred = adap->params.ofldq_wr_cred;
+ lld->crypto = adap->params.crypto;
lld->iscsi_iolen = MAXRXDATA_G(t4_read_reg(a
[crypto v3 2/2] chcr: Add support for Inline IPSec
register xfrmdev_ops callbacks, Send IPsec tunneled data
to HW for inline processing.
The driver use hardware crypto accelerator to encrypt and
generate ICV for the transmitted packet in Inline mode.
Signed-off-by: Atul Gupta
Signed-off-by: Harsh Jain
Signed-off-by: Ganesh Goudar
---
V2: Fixed the build warnings and created patch against cryptodev
to avoid possible merge conflicts
V3: Fixed a build warning and added flag for data packet
---
drivers/crypto/chelsio/Kconfig | 10 +
drivers/crypto/chelsio/Makefile | 1 +
drivers/crypto/chelsio/chcr_algo.c | 221 ++--
drivers/crypto/chelsio/chcr_algo.h | 15 -
drivers/crypto/chelsio/chcr_core.c | 14 +
drivers/crypto/chelsio/chcr_core.h | 38 ++
drivers/crypto/chelsio/chcr_crypto.h | 69 ++--
drivers/crypto/chelsio/chcr_ipsec.c | 660 +++
8 files changed, 860 insertions(+), 168 deletions(-)
create mode 100644 drivers/crypto/chelsio/chcr_ipsec.c
diff --git a/drivers/crypto/chelsio/Kconfig b/drivers/crypto/chelsio/Kconfig
index 3e104f5..51932c7 100644
--- a/drivers/crypto/chelsio/Kconfig
+++ b/drivers/crypto/chelsio/Kconfig
@@ -18,3 +18,13 @@ config CRYPTO_DEV_CHELSIO
To compile this driver as a module, choose M here: the module
will be called chcr.
+
+config CHELSIO_IPSEC_INLINE
+bool "Chelsio IPSec XFRM Tx crypto offload"
+depends on CHELSIO_T4
+ depends on CRYPTO_DEV_CHELSIO
+depends on XFRM_OFFLOAD
+depends on INET_ESP_OFFLOAD || INET6_ESP_OFFLOAD
+default n
+---help---
+ Enable support for IPSec Tx Inline.
diff --git a/drivers/crypto/chelsio/Makefile b/drivers/crypto/chelsio/Makefile
index bebdf06..eaecaf1 100644
--- a/drivers/crypto/chelsio/Makefile
+++ b/drivers/crypto/chelsio/Makefile
@@ -2,3 +2,4 @@ ccflags-y := -Idrivers/net/ethernet/chelsio/cxgb4
obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chcr.o
chcr-objs := chcr_core.o chcr_algo.o
+chcr-$(CONFIG_CHELSIO_IPSEC_INLINE) += chcr_ipsec.o
diff --git a/drivers/crypto/chelsio/chcr_algo.c
b/drivers/crypto/chelsio/chcr_algo.c
index 4eed717..b623368 100644
--- a/drivers/crypto/chelsio/chcr_algo.c
+++ b/drivers/crypto/chelsio/chcr_algo.c
@@ -73,6 +73,25 @@
#define IV AES_BLOCK_SIZE
+unsigned int sgl_ent_len[] = {0, 0, 16, 24, 40, 48, 64, 72, 88,
+ 96, 112, 120, 136, 144, 160, 168, 184,
+ 192, 208, 216, 232, 240, 256, 264, 280,
+ 288, 304, 312, 328, 336, 352, 360, 376};
+
+unsigned int dsgl_ent_len[] = {0, 32, 32, 48, 48, 64, 64, 80, 80,
+ 112, 112, 128, 128, 144, 144, 160, 160,
+ 192, 192, 208, 208, 224, 224, 240, 240,
+ 272, 272, 288, 288, 304, 304, 320, 320};
+
+static u32 round_constant[11] = {
+ 0x0100, 0x0200, 0x0400, 0x0800,
+ 0x1000, 0x2000, 0x4000, 0x8000,
+ 0x1B00, 0x3600, 0x6C00
+};
+
+static int chcr_handle_cipher_resp(struct ablkcipher_request *req,
+ unsigned char *input, int err);
+
static inline struct chcr_aead_ctx *AEAD_CTX(struct chcr_context *ctx)
{
return ctx->crypto_ctx->aeadctx;
@@ -108,18 +127,6 @@ static inline int is_ofld_imm(const struct sk_buff *skb)
return (skb->len <= SGE_MAX_WR_LEN);
}
-/*
- * sgl_len - calculates the size of an SGL of the given capacity
- * @n: the number of SGL entries
- * Calculates the number of flits needed for a scatter/gather list that
- * can hold the given number of entries.
- */
-static inline unsigned int sgl_len(unsigned int n)
-{
- n--;
- return (3 * n) / 2 + (n & 1) + 2;
-}
-
static int sg_nents_xlen(struct scatterlist *sg, unsigned int reqlen,
unsigned int entlen,
unsigned int skip)
@@ -183,30 +190,17 @@ static inline void chcr_handle_ahash_resp(struct
ahash_request *req,
}
out:
req->base.complete(&req->base, err);
+}
- }
-
-static inline void chcr_handle_aead_resp(struct aead_request *req,
-unsigned char *input,
-int err)
+static inline int get_aead_subtype(struct crypto_aead *aead)
{
- struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
- struct crypto_aead *tfm = crypto_aead_reqtfm(req);
- struct uld_ctx *u_ctx = ULD_CTX(a_ctx(tfm));
-
-
- chcr_aead_dma_unmap(&u_ctx->lldi.pdev->dev, req, reqctx->op);
- if (reqctx->b0_dma)
- dma_unmap_single(&u_ctx->lldi.pdev->dev, reqctx->b0_dma,
-reqctx->b0_len, DMA_BIDIRECTIONAL);
- if (reqctx->verify == VERIFY_SW) {
- chcr_verify_tag(req, input, &err);
- reqctx->verify = V
[crypto] chcr: fix a type cast error
fix a type cast error for queue descriptor Reported-by: Dan Carpenter Signed-off-by: Atul Gupta --- drivers/crypto/chelsio/chcr_ipsec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/chelsio/chcr_ipsec.c b/drivers/crypto/chelsio/chcr_ipsec.c index f90f991..a413156 100644 --- a/drivers/crypto/chelsio/chcr_ipsec.c +++ b/drivers/crypto/chelsio/chcr_ipsec.c @@ -428,7 +428,7 @@ inline void *copy_key_cpltx_pktxt(struct sk_buff *skb, memcpy(pos, sa_entry->key, left); memcpy(q->q.desc, sa_entry->key + left, key_len - left); - pos = q->q.desc + (key_len - left); + pos = (u8 *)q->q.desc + (key_len - left); } } /* Copy CPL TX PKT XT */ -- 1.8.3.1
[crypto 0/8] Chelsio inline TLS
RFC series for Chelsio Inline TLS driver (chtls.ko) Chtls driver use the available ULP infrastructure to register chtls as another ULP. Chtls use the TCP Sockets to transmit and receive TLS record. TCP proto_ops is extended to offload TLS record. T6 adapter provide the following features: -TLS record offload, add TLS header, encrypt data and transmit -TLS record receive and decrypt -TLS keys store -GCM crypto engine Atul Gupta (8): cxgb4: Inline TLS chcr: changes to chcr driver chtls: ulp for Inline TLS processing chtls: CPL handler definition chtls: Inline crypto request for Tx. chtls: TCB and Key program chtls: structure and macro definiton Kconfig Makefile drivers/crypto/chelsio/Kconfig | 10 + drivers/crypto/chelsio/Makefile|1 + drivers/crypto/chelsio/chcr_algo.h | 42 + drivers/crypto/chelsio/chcr_core.h | 55 +- drivers/crypto/chelsio/chtls/Makefile |4 + drivers/crypto/chelsio/chtls/chtls.h | 481 + drivers/crypto/chelsio/chtls/chtls_cm.c| 2048 drivers/crypto/chelsio/chtls/chtls_cm.h| 209 ++ drivers/crypto/chelsio/chtls/chtls_hw.c| 394 drivers/crypto/chelsio/chtls/chtls_io.c| 1866 ++ drivers/crypto/chelsio/chtls/chtls_main.c | 585 ++ drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c | 18 +- drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c| 32 +- drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h |7 + drivers/net/ethernet/chelsio/cxgb4/sge.c | 98 +- drivers/net/ethernet/chelsio/cxgb4/t4_msg.h| 121 +- drivers/net/ethernet/chelsio/cxgb4/t4_regs.h |2 + drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h | 165 +- include/uapi/linux/tls.h |1 + 19 files changed, 6120 insertions(+), 19 deletions(-) create mode 100644 drivers/crypto/chelsio/chtls/Makefile create mode 100644 drivers/crypto/chelsio/chtls/chtls.h create mode 100644 drivers/crypto/chelsio/chtls/chtls_cm.c create mode 100644 drivers/crypto/chelsio/chtls/chtls_cm.h create mode 100644 drivers/crypto/chelsio/chtls/chtls_hw.c create mode 100644 drivers/crypto/chelsio/chtls/chtls_io.c create mode 100644 drivers/crypto/chelsio/chtls/chtls_main.c -- 1.8.3.1
[crypto 1/8] cxgb4: Inline TLS
Add new uld driver for Inline TLS support. WR is defined to
submit crypto request to firmware.
Key area size is configured in hw-config file.
Signed-off-by: Atul Gupta
---
drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c | 18 ++-
drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c| 32 ++--
drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h | 7 +
drivers/net/ethernet/chelsio/cxgb4/sge.c | 98 +++-
drivers/net/ethernet/chelsio/cxgb4/t4_msg.h| 121 ++-
drivers/net/ethernet/chelsio/cxgb4/t4_regs.h | 2 +
drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h | 165 -
7 files changed, 425 insertions(+), 18 deletions(-)
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
index cf47183..cfc9210 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
@@ -2826,8 +2826,8 @@ static int meminfo_show(struct seq_file *seq, void *v)
"Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
"TDDP region:", "TPT region:", "STAG region:", "RQ region:",
"RQUDP region:", "PBL region:", "TXPBL region:",
- "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
- "On-chip queues:"
+ "TLSKey region:", "DBVFIFO region:", "ULPRX state:",
+ "ULPTX state:", "On-chip queues:"
};
int i, n;
@@ -2943,6 +2943,12 @@ static int meminfo_show(struct seq_file *seq, void *v)
ulp_region(RX_RQUDP);
ulp_region(RX_PBL);
ulp_region(TX_PBL);
+ if (adap->params.crypto & FW_CAPS_CONFIG_TLS_INLINE) {
+ ulp_region(RX_TLS_KEY);
+ } else {
+ md->base = 0;
+ md->idx = ARRAY_SIZE(region);
+ }
#undef ulp_region
md->base = 0;
md->idx = ARRAY_SIZE(region);
@@ -3098,6 +3104,14 @@ static int chcr_show(struct seq_file *seq, void *v)
atomic_read(&adap->chcr_stats.fallback));
seq_printf(seq, "IPSec PDU: %10u\n",
atomic_read(&adap->chcr_stats.ipsec_cnt));
+
+ seq_puts(seq, "\nChelsio Inline TLS Stats\n");
+ seq_printf(seq, "TLS PDU Tx: %u\n",
+ atomic_read(&adap->chcr_stats.tls_pdu_tx));
+ seq_printf(seq, "TLS PDU Rx: %u\n",
+ atomic_read(&adap->chcr_stats.tls_pdu_rx));
+ seq_printf(seq, "TLS Keys (DDR) Count: %u\n",
+ atomic_read(&adap->chcr_stats.tls_key));
return 0;
}
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
index 05a4abf..60eb18b 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
@@ -4086,18 +4086,32 @@ static int adap_init0(struct adapter *adap)
adap->num_ofld_uld += 2;
}
if (caps_cmd.cryptocaps) {
- /* Should query params here...TODO */
- params[0] = FW_PARAM_PFVF(NCRYPTO_LOOKASIDE);
- ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2,
- params, val);
- if (ret < 0) {
- if (ret != -EINVAL)
+ if (ntohs(caps_cmd.cryptocaps) &
+ FW_CAPS_CONFIG_CRYPTO_LOOKASIDE) {
+ params[0] = FW_PARAM_PFVF(NCRYPTO_LOOKASIDE);
+ ret = t4_query_params(adap, adap->mbox, adap->pf, 0,
+ 2, params, val);
+ if (ret < 0) {
+ if (ret != -EINVAL)
+ goto bye;
+ } else {
+ adap->vres.ncrypto_fc = val[0];
+ }
+ adap->num_ofld_uld += 1;
+ }
+ if (ntohs(caps_cmd.cryptocaps) &
+ FW_CAPS_CONFIG_TLS_INLINE) {
+ params[0] = FW_PARAM_PFVF(TLS_START);
+ params[1] = FW_PARAM_PFVF(TLS_END);
+ ret = t4_query_params(adap, adap->mbox, adap->pf, 0,
+ 2, params, val);
+ if (ret < 0)
goto bye;
- } else {
- adap->vres.ncrypto_fc = val[0];
+ adap->vres.key.start = val[0];
+ adap->vres.key.size = val[1] - val[0] + 1;
+ ad
[crypto 2/8] chcr: changes to chcr driver
Define the Macro for TLS Key context
Signed-off-by: Atul Gupta
---
drivers/crypto/chelsio/chcr_algo.h | 42 +
drivers/crypto/chelsio/chcr_core.h | 55 +-
include/uapi/linux/tls.h | 1 +
3 files changed, 97 insertions(+), 1 deletion(-)
diff --git a/drivers/crypto/chelsio/chcr_algo.h
b/drivers/crypto/chelsio/chcr_algo.h
index d1673a5..f263cd4 100644
--- a/drivers/crypto/chelsio/chcr_algo.h
+++ b/drivers/crypto/chelsio/chcr_algo.h
@@ -86,6 +86,39 @@
KEY_CONTEXT_OPAD_PRESENT_M)
#define KEY_CONTEXT_OPAD_PRESENT_F KEY_CONTEXT_OPAD_PRESENT_V(1U)
+#define TLS_KEYCTX_RXFLIT_CNT_S 24
+#define TLS_KEYCTX_RXFLIT_CNT_V(x) ((x) << TLS_KEYCTX_RXFLIT_CNT_S)
+
+#define TLS_KEYCTX_RXPROT_VER_S 20
+#define TLS_KEYCTX_RXPROT_VER_M 0xf
+#define TLS_KEYCTX_RXPROT_VER_V(x) ((x) << TLS_KEYCTX_RXPROT_VER_S)
+
+#define TLS_KEYCTX_RXCIPH_MODE_S 16
+#define TLS_KEYCTX_RXCIPH_MODE_M 0xf
+#define TLS_KEYCTX_RXCIPH_MODE_V(x) ((x) << TLS_KEYCTX_RXCIPH_MODE_S)
+
+#define TLS_KEYCTX_RXAUTH_MODE_S 12
+#define TLS_KEYCTX_RXAUTH_MODE_M 0xf
+#define TLS_KEYCTX_RXAUTH_MODE_V(x) ((x) << TLS_KEYCTX_RXAUTH_MODE_S)
+
+#define TLS_KEYCTX_RXCIAU_CTRL_S 11
+#define TLS_KEYCTX_RXCIAU_CTRL_V(x) ((x) << TLS_KEYCTX_RXCIAU_CTRL_S)
+
+#define TLS_KEYCTX_RX_SEQCTR_S 9
+#define TLS_KEYCTX_RX_SEQCTR_M 0x3
+#define TLS_KEYCTX_RX_SEQCTR_V(x) ((x) << TLS_KEYCTX_RX_SEQCTR_S)
+
+#define TLS_KEYCTX_RX_VALID_S 8
+#define TLS_KEYCTX_RX_VALID_V(x) ((x) << TLS_KEYCTX_RX_VALID_S)
+
+#define TLS_KEYCTX_RXCK_SIZE_S 3
+#define TLS_KEYCTX_RXCK_SIZE_M 0x7
+#define TLS_KEYCTX_RXCK_SIZE_V(x) ((x) << TLS_KEYCTX_RXCK_SIZE_S)
+
+#define TLS_KEYCTX_RXMK_SIZE_S 0
+#define TLS_KEYCTX_RXMK_SIZE_M 0x7
+#define TLS_KEYCTX_RXMK_SIZE_V(x) ((x) << TLS_KEYCTX_RXMK_SIZE_S)
+
#define CHCR_HASH_MAX_DIGEST_SIZE 64
#define CHCR_MAX_SHA_DIGEST_SIZE 64
@@ -176,6 +209,15 @@
KEY_CONTEXT_SALT_PRESENT_V(1) | \
KEY_CONTEXT_CTX_LEN_V((ctx_len)))
+#define FILL_KEY_CRX_HDR(ck_size, mk_size, d_ck, opad, ctx_len) \
+ htonl(TLS_KEYCTX_RXMK_SIZE_V(mk_size) | \
+ TLS_KEYCTX_RXCK_SIZE_V(ck_size) | \
+ TLS_KEYCTX_RX_VALID_V(1) | \
+ TLS_KEYCTX_RX_SEQCTR_V(3) | \
+ TLS_KEYCTX_RXAUTH_MODE_V(4) | \
+ TLS_KEYCTX_RXCIPH_MODE_V(2) | \
+ TLS_KEYCTX_RXFLIT_CNT_V((ctx_len)))
+
#define FILL_WR_OP_CCTX_SIZE \
htonl( \
FW_CRYPTO_LOOKASIDE_WR_OPCODE_V( \
diff --git a/drivers/crypto/chelsio/chcr_core.h
b/drivers/crypto/chelsio/chcr_core.h
index 3c29ee0..77056a9 100644
--- a/drivers/crypto/chelsio/chcr_core.h
+++ b/drivers/crypto/chelsio/chcr_core.h
@@ -65,10 +65,58 @@
struct _key_ctx {
__be32 ctx_hdr;
u8 salt[MAX_SALT];
- __be64 reserverd;
+ __be64 iv_to_auth;
unsigned char key[0];
};
+#define KEYCTX_TX_WR_IV_S 55
+#define KEYCTX_TX_WR_IV_M 0x1ffULL
+#define KEYCTX_TX_WR_IV_V(x) ((x) << KEYCTX_TX_WR_IV_S)
+#define KEYCTX_TX_WR_IV_G(x) \
+ (((x) >> KEYCTX_TX_WR_IV_S) & KEYCTX_TX_WR_IV_M)
+
+#define KEYCTX_TX_WR_AAD_S 47
+#define KEYCTX_TX_WR_AAD_M 0xffULL
+#define KEYCTX_TX_WR_AAD_V(x) ((x) << KEYCTX_TX_WR_AAD_S)
+#define KEYCTX_TX_WR_AAD_G(x) (((x) >> KEYCTX_TX_WR_AAD_S) & \
+ KEYCTX_TX_WR_AAD_M)
+
+#define KEYCTX_TX_WR_AADST_S 39
+#define KEYCTX_TX_WR_AADST_M 0xffULL
+#define KEYCTX_TX_WR_AADST_V(x) ((x) << KEYCTX_TX_WR_AADST_S)
+#define KEYCTX_TX_WR_AADST_G(x) \
+ (((x) >> KEYCTX_TX_WR_AADST_S) & KEYCTX_TX_WR_AADST_M)
+
+#define KEYCTX_TX_WR_CIPHER_S 30
+#define KEYCTX_TX_WR_CIPHER_M 0x1ffULL
+#define KEYCTX_TX_WR_CIPHER_V(x) ((x) << KEYCTX_TX_WR_CIPHER_S)
+#define KEYCTX_TX_WR_CIPHER_G(x) \
+ (((x) >> KEYCTX_TX_WR_CIPHER_S) & KEYCTX_TX_WR_CIPHER_M)
+
+#define KEYCTX_TX_WR_CIPHERST_S 23
+#define KEYCTX_TX_WR_CIPHERST_M 0x7f
+#define KEYCTX_TX_WR_CIPHERST_V(x) ((x) << KEYCTX_TX_WR_CIPHERST_S)
+#define KEYCTX_TX_WR_CIPHERST_G(x) \
+ (((x) >> KEYCTX_TX_WR_CIPHERST_S) & KEYCTX_TX_WR_CIPHERST_M)
+
+#define KEYCTX_TX_WR_AUTH_S 14
+#define KEYCTX_TX_WR_AUTH_M 0x1ff
+#define KEYCTX_TX_WR_AUTH_V(x) ((x) << KEYCTX_TX_WR_AUTH_S)
+#define KEYCTX_TX_WR_AUTH_G(x) \
+ (((x) >> KEYCTX_TX_WR_AUTH_S) & KEYCTX_TX_WR_AUTH_M)
+
+#define KEYCTX_TX_WR_AUTHST_S 7
+#define KEYCTX_TX_WR_AUTHST_M 0x7f
+#define KEYCTX_TX_WR_AUTHST_V(x) ((x) << KEYCTX_TX_WR_AUTHST_S)
+#define KEYCTX_TX_WR_AUTHST_G(x) \
+ (((x) >> KEYCTX_TX_WR_AUTHST_S) & KEYCTX_TX_WR_AUTHST_M)
+
+#define KEYCTX_TX_WR_AUTHIN_S 0
+#define KEYCTX_TX_WR_AUTHIN_M 0x7f
+#define KEYCTX_TX_WR_AUTHIN_V(x) ((x) << KEYCTX_TX_WR_AUTHIN_S)
+#define KEYCTX_TX_WR_AUTHIN_G(x) \
+ (((x) >>
[crypto 3/8] chtls: ulp for Inline TLS processing
Register chtls as another tcp ULP, Based on a similar infrastructure
in tcp_cong. proto_ops are defined to handle CPL to send/receive
crypto request to hw.
Signed-off-by: Atul Gupta
---
drivers/crypto/chelsio/chtls/chtls_main.c | 585 ++
1 file changed, 585 insertions(+)
create mode 100644 drivers/crypto/chelsio/chtls/chtls_main.c
diff --git a/drivers/crypto/chelsio/chtls/chtls_main.c
b/drivers/crypto/chelsio/chtls/chtls_main.c
new file mode 100644
index 000..e951b4e
--- /dev/null
+++ b/drivers/crypto/chelsio/chtls/chtls_main.c
@@ -0,0 +1,585 @@
+/*
+ * Copyright (c) 2017 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Written by: Atul Gupta (atul.gu...@chelsio.com)
+ */
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "chtls.h"
+#include "chtls_cm.h"
+
+#define DRV_NAME "chtls"
+
+/*
+ * chtls device management
+ * maintains a list of the chtls devices
+ */
+static LIST_HEAD(cdev_list);
+static DEFINE_MUTEX(cdev_mutex);
+static DEFINE_MUTEX(cdev_list_lock);
+
+static struct proto chtls_base_prot;
+static struct proto chtls_cpl_prot;
+static DEFINE_MUTEX(notify_mutex);
+static RAW_NOTIFIER_HEAD(listen_notify_list);
+struct request_sock_ops chtls_rsk_ops;
+static uint send_page_order = (14 - PAGE_SHIFT < 0) ? 0 : 14 - PAGE_SHIFT;
+
+int register_listen_notifier(struct notifier_block *nb)
+{
+ int err;
+
+ mutex_lock(¬ify_mutex);
+ err = raw_notifier_chain_register(&listen_notify_list, nb);
+ mutex_unlock(¬ify_mutex);
+ return err;
+}
+
+int unregister_listen_notifier(struct notifier_block *nb)
+{
+ int err;
+
+ mutex_lock(¬ify_mutex);
+ err = raw_notifier_chain_unregister(&listen_notify_list, nb);
+ mutex_unlock(¬ify_mutex);
+ return err;
+}
+
+int listen_notify_handler(struct notifier_block *this,
+ unsigned long event, void *data)
+{
+ struct sock *sk = data;
+ struct chtls_dev *cdev;
+ int ret = NOTIFY_DONE;
+
+ switch (event) {
+ case CHTLS_LISTEN_START:
+ case CHTLS_LISTEN_STOP:
+ mutex_lock(&cdev_list_lock);
+ list_for_each_entry(cdev, &cdev_list, list) {
+ if (event == CHTLS_LISTEN_START)
+ ret = chtls_listen_start(cdev, sk);
+ else
+ chtls_listen_stop(cdev, sk);
+ }
+ mutex_unlock(&cdev_list_lock);
+ break;
+ }
+ return ret;
+}
+
+static struct notifier_block listen_notifier = {
+ .notifier_call = listen_notify_handler
+};
+
+static int listen_backlog_rcv(struct sock *sk, struct sk_buff *skb)
+{
+ if (likely(skb_transport_header(skb) != skb_network_header(skb)))
+ return tcp_v4_do_rcv(sk, skb);
+ BLOG_SKB_CB(skb)->backlog_rcv(sk, skb);
+ return 0;
+}
+
+static int chtls_start_listen(struct sock *sk)
+{
+ int err;
+
+ if (sk->sk_protocol != IPPROTO_TCP)
+ return -EPROTONOSUPPORT;
+
+ if (sk->sk_family == PF_INET &&
+ LOOPBACK(inet_sk(sk)->inet_rcv_saddr))
+ return -EADDRNOTAVAIL;
+
+ sk->sk_backlog_rcv = listen_backlog_rcv;
+ mutex_lock(¬ify_mutex);
+ err = raw_notifier_call_chain(&listen_notify_list, 0, sk);
+ mutex_unlock(¬ify_mutex);
+ return err;
+}
+
+static int chtls_hash(struct sock *sk)
+{
+ int err;
+
+ err = tcp_prot.hash(sk);
+ if (sk->sk_state == TCP_LISTEN)
+ err |= chtls_start_listen(sk);
+
+ if (err)
+ tcp_prot.unhash(sk);
+ return err;
+}
+
+int chtls_stop_listen(struct sock *sk)
+{
+ if (sk->sk_protocol != IPPROTO_TCP)
+ return -EPROTONOSUPPORT;
+
+ mutex_lock(¬ify_mutex);
+ raw_notifier_call_chain(&listen_notify_list, 1, sk);
+ mutex_unlock(¬ify_mutex);
+ return 0;
+}
+
+static void chtls_unhash(struct sock *sk)
+{
+ if (sk->sk_state == TCP_LISTEN)
+ chtls_stop_listen(sk);
+ tcp_prot.unhash(sk);
+}
+
+static void chtls_lsk_close(struct sock *sk, long timeout)
+{
+ struct tls_context *ctx = tls_get_ctx(sk);
+ void (*sk_proto_close)(struct sock *sk, long timeout);
+
+ lock_sock(sk);
+ sk_proto_close = ctx->sk_proto_close;
+ kfree(ctx);
+
+ release_sock(sk);
+ sk_proto_close(sk, timeout);
+}
+
+static void process_deferq(struct work_struct *task_param)
+{
+ struct sk_buff *skb;
+ struct chtls_dev *cdev = container_of(task_param,
+ struct chtls_dev, deferq_task
[crypto 5/8] chtls: Inline crypto request for Tx.
TLS handler for record transmit and receive.
Create Inline TLS work request
Signed-off-by: Atul Gupta
---
drivers/crypto/chelsio/chtls/chtls_io.c | 1866 +++
1 file changed, 1866 insertions(+)
create mode 100644 drivers/crypto/chelsio/chtls/chtls_io.c
diff --git a/drivers/crypto/chelsio/chtls/chtls_io.c
b/drivers/crypto/chelsio/chtls/chtls_io.c
new file mode 100644
index 000..b63fb78
--- /dev/null
+++ b/drivers/crypto/chelsio/chtls/chtls_io.c
@@ -0,0 +1,1866 @@
+/*
+ * Copyright (c) 2017 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Written by: Atul Gupta (atul.gu...@chelsio.com)
+ */
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "chtls.h"
+#include "chtls_cm.h"
+
+static bool is_tls_hw(struct chtls_sock *csk)
+{
+ return csk->tlshws.ofld;
+}
+
+static bool is_tls_rx(struct chtls_sock *csk)
+{
+ return (csk->tlshws.rxkey >= 0);
+}
+
+static bool is_tls_tx(struct chtls_sock *csk)
+{
+ return (csk->tlshws.txkey >= 0);
+}
+
+static bool is_tls_skb(struct chtls_sock *csk, const struct sk_buff *skb)
+{
+ return (is_tls_hw(csk) && skb_ulp_tls_skb_flags(skb));
+}
+
+static int key_size(void *sk)
+{
+ return 16; /* Key on DDR */
+}
+
+#define ceil(x, y) \
+ ({ unsigned long __x = (x), __y = (y); (__x + __y - 1) / __y; })
+
+static int data_sgl_len(const struct sk_buff *skb)
+{
+ unsigned int cnt;
+
+ cnt = skb_shinfo(skb)->nr_frags;
+ return (sgl_len(cnt) * 8);
+}
+
+static int nos_ivs(struct sock *sk, unsigned int size)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+
+ return ceil(size, csk->tlshws.mfs);
+}
+
+#define TLS_WR_CPL_LEN \
+ (sizeof(struct fw_tlstx_data_wr) + \
+ sizeof(struct cpl_tx_tls_sfo))
+
+static int is_ivs_imm(struct sock *sk, const struct sk_buff *skb)
+{
+ int ivs_size = nos_ivs(sk, skb->len) * CIPHER_BLOCK_SIZE;
+ int hlen = TLS_WR_CPL_LEN + data_sgl_len(skb);
+
+ if ((hlen + key_size(sk) + ivs_size) <
+ MAX_IMM_OFLD_TX_DATA_WR_LEN) {
+ ULP_SKB_CB(skb)->ulp.tls.iv = 1;
+ return 1;
+ }
+ ULP_SKB_CB(skb)->ulp.tls.iv = 0;
+ return 0;
+}
+
+static int max_ivs_size(struct sock *sk, int size)
+{
+ return (nos_ivs(sk, size) * CIPHER_BLOCK_SIZE);
+}
+
+static int ivs_size(struct sock *sk, const struct sk_buff *skb)
+{
+ return (is_ivs_imm(sk, skb) ? (nos_ivs(sk, skb->len) *
+CIPHER_BLOCK_SIZE) : 0);
+}
+
+static int flowc_wr_credits(int nparams, int *flowclenp)
+{
+ int flowclen16, flowclen;
+
+ flowclen = offsetof(struct fw_flowc_wr, mnemval[nparams]);
+ flowclen16 = DIV_ROUND_UP(flowclen, 16);
+ flowclen = flowclen16 * 16;
+
+ if (flowclenp)
+ *flowclenp = flowclen;
+
+ return flowclen16;
+}
+
+struct sk_buff *create_flowc_wr_skb(struct sock *sk,
+ struct fw_flowc_wr *flowc,
+ int flowclen)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+ struct sk_buff *skb;
+
+ skb = alloc_skb(flowclen, GFP_ATOMIC);
+ if (!skb)
+ return NULL;
+
+ memcpy(__skb_put(skb, flowclen), flowc, flowclen);
+ set_queue(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA, sk);
+
+ return skb;
+}
+
+static int send_flowc_wr(struct sock *sk, struct fw_flowc_wr *flowc,
+int flowclen)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+ struct tcp_sock *tp = tcp_sk(sk);
+ bool syn_sent = (sk->sk_state == TCP_SYN_SENT);
+ int flowclen16 = flowclen / 16;
+ struct sk_buff *skb;
+
+ if (csk_flag(sk, CSK_TX_DATA_SENT)) {
+ skb = create_flowc_wr_skb(sk, flowc, flowclen);
+ if (!skb)
+ return -ENOMEM;
+
+ if (syn_sent)
+ __skb_queue_tail(&csk->ooo_queue, skb);
+ else
+ skb_entail(sk, skb,
+ ULPCB_FLAG_NO_HDR | ULPCB_FLAG_NO_APPEND);
+ return 0;
+ }
+
+ if (!syn_sent) {
+ int ret;
+
+ ret = cxgb4_immdata_send(csk->egress_dev,
+csk->txq_idx,
+flowc, flowclen);
+ if (!ret)
+ return flowclen16;
+ }
+ skb = create_flowc_wr_skb(sk, flowc, flowclen);
+ if (!skb)
+ return -ENOMEM;
+ send_or_defer(sk, tp, skb, 0);
[crypto 4/8] chtls: CPL handler definition
CPL handlers for TLS session, record transmit and receive
Signed-off-by: Atul Gupta
---
drivers/crypto/chelsio/chtls/chtls_cm.c | 2048 +++
1 file changed, 2048 insertions(+)
create mode 100644 drivers/crypto/chelsio/chtls/chtls_cm.c
diff --git a/drivers/crypto/chelsio/chtls/chtls_cm.c
b/drivers/crypto/chelsio/chtls/chtls_cm.c
new file mode 100644
index 000..ea1c301
--- /dev/null
+++ b/drivers/crypto/chelsio/chtls/chtls_cm.c
@@ -0,0 +1,2048 @@
+/*
+ * Copyright (c) 2017 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Written by: Atul Gupta (atul.gu...@chelsio.com)
+ */
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "chtls.h"
+#include "chtls_cm.h"
+
+extern struct request_sock_ops chtls_rsk_ops;
+static void (*tcp_time_wait_p)(struct sock *sk, int state, int timeo);
+
+/*
+ * State transitions and actions for close. Note that if we are in SYN_SENT
+ * we remain in that state as we cannot control a connection while it's in
+ * SYN_SENT; such connections are allowed to establish and are then aborted.
+ */
+static unsigned char new_state[16] = {
+ /* current state: new state: action: */
+ /* (Invalid) */ TCP_CLOSE,
+ /* TCP_ESTABLISHED */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
+ /* TCP_SYN_SENT*/ TCP_SYN_SENT,
+ /* TCP_SYN_RECV*/ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
+ /* TCP_FIN_WAIT1 */ TCP_FIN_WAIT1,
+ /* TCP_FIN_WAIT2 */ TCP_FIN_WAIT2,
+ /* TCP_TIME_WAIT */ TCP_CLOSE,
+ /* TCP_CLOSE */ TCP_CLOSE,
+ /* TCP_CLOSE_WAIT */ TCP_LAST_ACK | TCP_ACTION_FIN,
+ /* TCP_LAST_ACK*/ TCP_LAST_ACK,
+ /* TCP_LISTEN */ TCP_CLOSE,
+ /* TCP_CLOSING */ TCP_CLOSING,
+};
+
+static struct chtls_sock *chtls_sock_create(struct chtls_dev *cdev)
+{
+ struct chtls_sock *csk = kzalloc(sizeof(*csk), GFP_NOIO);
+
+ if (!csk)
+ return NULL;
+
+ csk->txdata_skb_cache = alloc_skb(TXDATA_SKB_LEN, GFP_ATOMIC);
+ if (!csk->txdata_skb_cache) {
+ kfree(csk);
+ return NULL;
+ }
+
+ kref_init(&csk->kref);
+ csk->cdev = cdev;
+ skb_queue_head_init(&csk->txq);
+ csk->wr_skb_head = NULL;
+ csk->wr_skb_tail = NULL;
+ csk->mss = MAX_MSS;
+ csk->tlshws.ofld = 1;
+ csk->tlshws.txkey = -1;
+ csk->tlshws.rxkey = -1;
+ csk->tlshws.mfs = TLS_MFS;
+ skb_queue_head_init(&csk->tlshws.sk_recv_queue);
+ return csk;
+}
+
+void chtls_sock_release(struct kref *ref)
+{
+ struct chtls_sock *csk =
+ container_of(ref, struct chtls_sock, kref);
+
+ kfree(csk);
+}
+
+void get_tcp_symbol(void)
+{
+ tcp_time_wait_p = (void *)kallsyms_lookup_name("tcp_time_wait");
+ if (!tcp_time_wait_p)
+ pr_info("could not locate tcp_time_wait");
+}
+
+static struct net_device *chtls_ipv4_netdev(struct chtls_dev *cdev,
+ struct sock *sk)
+{
+ struct net_device *ndev = cdev->ports[0];
+
+ if (likely(!inet_sk(sk)->inet_rcv_saddr))
+ return ndev;
+
+ ndev = ip_dev_find(&init_net, inet_sk(sk)->inet_rcv_saddr);
+ if (!ndev)
+ return NULL;
+
+ if (is_vlan_dev(ndev))
+ return vlan_dev_real_dev(ndev);
+ return ndev;
+}
+
+static void assign_rxopt(struct sock *sk, unsigned int opt)
+{
+ const struct chtls_dev *cdev;
+ struct tcp_sock *tp = tcp_sk(sk);
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+
+ cdev = csk->cdev;
+ tp->tcp_header_len = sizeof(struct tcphdr);
+ tp->rx_opt.mss_clamp = cdev->mtus[TCPOPT_MSS_G(opt)] - 40;
+ tp->mss_cache= tp->rx_opt.mss_clamp;
+ tp->rx_opt.tstamp_ok = TCPOPT_TSTAMP_G(opt);
+ tp->rx_opt.snd_wscale= TCPOPT_SACK_G(opt);
+ tp->rx_opt.wscale_ok = TCPOPT_WSCALE_OK_G(opt);
+ SND_WSCALE(tp) = TCPOPT_SND_WSCALE_G(opt);
+ if (!tp->rx_opt.wscale_ok)
+ tp->rx_opt.rcv_wscale = 0;
+ if (tp->rx_opt.tstamp_ok) {
+ tp->tcp_header_len += TCPOLEN_TSTAMP_ALIGNED;
+ tp->rx_opt.mss_clamp -= TCPOLEN_TSTAMP_ALIGNED;
+ } else if (csk->opt2 & TSTAMPS_EN_F) {
+ csk->opt2 &= ~TSTAMPS_EN_F;
+ csk->mtu_idx = TCPOPT_MSS_G(opt);
+ }
+}
+
+static void chtls_purge_rcv_queue(struct sock *sk)
+{
+ str
[crypto 7/8] chtls: structure and macro definiton
Inline TLS state, connection management. Support macros definition.
Signed-off-by: Atul Gupta
---
drivers/crypto/chelsio/chtls/Makefile | 4 +
drivers/crypto/chelsio/chtls/chtls.h| 481
drivers/crypto/chelsio/chtls/chtls_cm.h | 209 ++
3 files changed, 694 insertions(+)
create mode 100644 drivers/crypto/chelsio/chtls/Makefile
create mode 100644 drivers/crypto/chelsio/chtls/chtls.h
create mode 100644 drivers/crypto/chelsio/chtls/chtls_cm.h
diff --git a/drivers/crypto/chelsio/chtls/Makefile
b/drivers/crypto/chelsio/chtls/Makefile
new file mode 100644
index 000..df13795
--- /dev/null
+++ b/drivers/crypto/chelsio/chtls/Makefile
@@ -0,0 +1,4 @@
+ccflags-y := -Idrivers/net/ethernet/chelsio/cxgb4 -Idrivers/crypto/chelsio/
+
+obj-$(CONFIG_CRYPTO_DEV_CHELSIO_TLS) += chtls.o
+chtls-objs := chtls_main.o chtls_cm.o chtls_io.o chtls_hw.o
diff --git a/drivers/crypto/chelsio/chtls/chtls.h
b/drivers/crypto/chelsio/chtls/chtls.h
new file mode 100644
index 000..266fef7c
--- /dev/null
+++ b/drivers/crypto/chelsio/chtls/chtls.h
@@ -0,0 +1,481 @@
+/*
+ * Copyright (c) 2016 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef __CHTLS_H__
+#define __CHTLS_H__
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "t4fw_api.h"
+#include "t4_msg.h"
+#include "cxgb4.h"
+#include "cxgb4_uld.h"
+#include "l2t.h"
+#include "chcr_algo.h"
+#include "chcr_core.h"
+#include "chcr_crypto.h"
+
+#define CIPHER_BLOCK_SIZE 16
+#define MAX_IVS_PAGE256
+#define TLS_KEY_CONTEXT_SZ 64
+#define TLS_HEADER_LENGTH 5
+#define SCMD_CIPH_MODE_AES_GCM 2
+#define GCM_TAG_SIZE16
+#define AEAD_EXPLICIT_DATA_SIZE 8
+/* Any MFS size should work and come from openssl */
+#define TLS_MFS16384
+
+#define SOCK_INLINE (31)
+#define RSS_HDR sizeof(struct rss_header)
+
+enum {
+ CHTLS_KEY_CONTEXT_DSGL,
+ CHTLS_KEY_CONTEXT_IMM,
+ CHTLS_KEY_CONTEXT_DDR,
+};
+
+enum {
+ CHTLS_LISTEN_START,
+ CHTLS_LISTEN_STOP,
+};
+
+/* Flags for return value of CPL message handlers */
+enum {
+ CPL_RET_BUF_DONE = 1, /* buffer processing done */
+ CPL_RET_BAD_MSG = 2,/* bad CPL message */
+ CPL_RET_UNKNOWN_TID = 4 /* unexpected unknown TID */
+};
+
+#define TLS_RCV_ST_READ_HEADER 0xF0
+#define TLS_RCV_ST_READ_BODY0xF1
+#define TLS_RCV_ST_READ_DONE0xF2
+#define TLS_RCV_ST_READ_NB 0xF3
+
+#define RSPQ_HASH_BITS 5
+#define LISTEN_INFO_HASH_SIZE 32
+struct listen_info {
+ struct listen_info *next; /* Link to next entry */
+ struct sock *sk; /* The listening socket */
+ unsigned int stid; /* The server TID */
+};
+
+enum {
+ T4_LISTEN_START_PENDING,
+ T4_LISTEN_STARTED
+};
+
+enum csk_flags {
+ CSK_CALLBACKS_CHKD, /* socket callbacks have been sanitized */
+ CSK_ABORT_REQ_RCVD, /* received one ABORT_REQ_RSS message */
+ CSK_TX_MORE_DATA, /* sending ULP data; don't set SHOVE bit */
+ CSK_TX_WAIT_IDLE, /* suspend Tx until in-flight data is ACKed */
+ CSK_ABORT_SHUTDOWN, /* shouldn't send more abort requests */
+ CSK_ABORT_RPL_PENDING, /* expecting an abort reply */
+ CSK_CLOSE_CON_REQUESTED,/* we've sent a close_conn_req */
+ CSK_TX_DATA_SENT, /* sent a TX_DATA WR on this connection */
+ CSK_TX_FAILOVER,/* Tx traffic failing over */
+ CSK_UPDATE_RCV_WND, /* Need to update rcv window */
+ CSK_RST_ABORTED,/* outgoing RST was aborted */
+ CSK_TLS_HANDSHK,/* TLS Handshake */
+};
+
+struct listen_ctx {
+ struct sock *lsk;
+ struct chtls_dev *cdev;
+ u32 state;
+};
+
+struct key_map {
+ unsigned long *addr;
+ unsigned int start;
+ unsigned int available;
+ unsigned int size;
+ spinlock_t lock; /* lock for key id request from map */
+} __packed;
+
+struct tls_scmd {
+ __be32 seqno_numivs;
+ __be32 ivgen_hdrlen;
+};
+
+struct chtls_dev {
+ struct list_head list;
+ struct cxgb4_lld_info *lldi;
+ struct pci_dev *pdev;
+ struct listen_info *listen_hash_tab[LISTEN_INFO_HASH_SIZE];
+ spinlock_t listen_lock; /* lock for listen list */
+ struct net_device **ports;
+ struct tid_info *tids;
+ unsigned int pfvf;
+ const unsigned short *mtus;
+
+ spinlock_t aidr_lock cacheline_aligned_in_smp;
+ struct idr aidr; /* ATID id sp
[crypto 6/8] chtls: TCB and Key program
program the tx and rx key on chip.
Signed-off-by: Atul Gupta
---
drivers/crypto/chelsio/chtls/chtls_hw.c | 394
1 file changed, 394 insertions(+)
create mode 100644 drivers/crypto/chelsio/chtls/chtls_hw.c
diff --git a/drivers/crypto/chelsio/chtls/chtls_hw.c
b/drivers/crypto/chelsio/chtls/chtls_hw.c
new file mode 100644
index 000..5e65aa2
--- /dev/null
+++ b/drivers/crypto/chelsio/chtls/chtls_hw.c
@@ -0,0 +1,394 @@
+/*
+ * Copyright (c) 2017 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Written by: Atul Gupta (atul.gu...@chelsio.com)
+ */
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "chtls.h"
+#include "chtls_cm.h"
+
+static void __set_tcb_field_direct(struct chtls_sock *csk,
+ struct cpl_set_tcb_field *req, u16 word,
+ u64 mask, u64 val, u8 cookie, int no_reply)
+{
+ struct ulptx_idata *sc;
+
+ INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, csk->tid);
+ req->wr.wr_mid |= htonl(FW_WR_FLOWID_V(csk->tid));
+ req->reply_ctrl = htons(NO_REPLY_V(no_reply) |
+ QUEUENO_V(csk->rss_qid));
+ req->word_cookie = htons(TCB_WORD(word) | TCB_COOKIE_V(cookie));
+ req->mask = cpu_to_be64(mask);
+ req->val = cpu_to_be64(val);
+ sc = (struct ulptx_idata *)(req + 1);
+ sc->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP));
+ sc->len = htonl(0);
+}
+
+void __set_tcb_field(struct sock *sk, struct sk_buff *skb, u16 word,
+u64 mask, u64 val, u8 cookie, int no_reply)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+ struct cpl_set_tcb_field *req;
+ struct ulptx_idata *sc;
+ unsigned int wrlen = roundup(sizeof(*req) + sizeof(*sc), 16);
+
+ req = (struct cpl_set_tcb_field *)__skb_put(skb, wrlen);
+ __set_tcb_field_direct(csk, req, word, mask, val, cookie, no_reply);
+ set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id);
+}
+
+static int chtls_set_tcb_field(struct sock *sk, u16 word, u64 mask, u64 val)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+ struct sk_buff *skb;
+ struct cpl_set_tcb_field *req;
+ struct ulptx_idata *sc;
+ unsigned int wrlen = roundup(sizeof(*req) + sizeof(*sc), 16);
+ unsigned int credits_needed = DIV_ROUND_UP(wrlen, 16);
+
+ skb = alloc_skb(wrlen, GFP_ATOMIC);
+ if (!skb)
+ return -ENOMEM;
+
+ __set_tcb_field(sk, skb, word, mask, val, 0, 1);
+ set_queue(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA, sk);
+ csk->wr_credits -= credits_needed;
+ csk->wr_unacked += credits_needed;
+ enqueue_wr(csk, skb);
+ cxgb4_ofld_send(csk->egress_dev, skb);
+ return 0;
+}
+
+/*
+ * Set one of the t_flags bits in the TCB.
+ */
+int chtls_set_tcb_tflag(struct sock *sk, unsigned int bit_pos, int val)
+{
+ return chtls_set_tcb_field(sk, 1, 1ULL << bit_pos,
+ val << bit_pos);
+}
+
+static int chtls_set_tcb_keyid(struct sock *sk, int keyid)
+{
+ return chtls_set_tcb_field(sk, 31, 0xULL, keyid);
+}
+
+static int chtls_set_tcb_seqno(struct sock *sk)
+{
+ return chtls_set_tcb_field(sk, 28, ~0ULL, 0);
+}
+
+static int chtls_set_tcb_quiesce(struct sock *sk, int val)
+{
+ return chtls_set_tcb_field(sk, 1, (1ULL << TF_RX_QUIESCE_S),
+ TF_RX_QUIESCE_V(val));
+}
+
+static void *chtls_alloc_mem(unsigned long size)
+{
+ void *p = kmalloc(size, GFP_KERNEL);
+
+ if (!p)
+ p = vmalloc(size);
+ if (p)
+ memset(p, 0, size);
+ return p;
+}
+
+static void chtls_free_mem(void *addr)
+{
+ unsigned long p = (unsigned long)addr;
+
+ if (p >= VMALLOC_START && p < VMALLOC_END)
+ vfree(addr);
+ else
+ kfree(addr);
+}
+
+/* TLS Key bitmap processing */
+int chtls_init_kmap(struct chtls_dev *cdev, struct cxgb4_lld_info *lldi)
+{
+ unsigned int num_key_ctx, bsize;
+
+ num_key_ctx = (lldi->vr->key.size / TLS_KEY_CONTEXT_SZ);
+ bsize = BITS_TO_LONGS(num_key_ctx);
+
+ cdev->kmap.size = num_key_ctx;
+ cdev->kmap.available = bsize;
+ cdev->kmap.addr = chtls_alloc_mem(sizeof(*cdev->kmap.addr) *
+ bsize);
+ if (!cdev->kmap.addr)
+ return -1;
+
+ cdev->kmap.start = lldi->vr->key.start;
+ spin_lock_init(&cdev->kmap.lock);
+ return 0;
+}
+
+void chtls_free_kmap(struct chtls_dev *cdev)
[crypto 8/8] Kconfig Makefile
Entry for Inline TLS as another driver dependent on cxgb4 and chcr Signed-off-by: Atul Gupta --- drivers/crypto/chelsio/Kconfig | 10 ++ drivers/crypto/chelsio/Makefile | 1 + 2 files changed, 11 insertions(+) diff --git a/drivers/crypto/chelsio/Kconfig b/drivers/crypto/chelsio/Kconfig index 51932c7..686d246 100644 --- a/drivers/crypto/chelsio/Kconfig +++ b/drivers/crypto/chelsio/Kconfig @@ -28,3 +28,13 @@ config CHELSIO_IPSEC_INLINE default n ---help--- Enable support for IPSec Tx Inline. + +config CRYPTO_DEV_CHELSIO_TLS +tristate "Chelsio Crypto Inline TLS Driver" +depends on CHELSIO_T4 +select CRYPTO_DEV_CHELSIO +---help--- + Support Chelsio Inline TLS with Chelsio crypto accelerator. + + To compile this driver as a module, choose M here: the module + will be called chtls. diff --git a/drivers/crypto/chelsio/Makefile b/drivers/crypto/chelsio/Makefile index eaecaf1..639e571 100644 --- a/drivers/crypto/chelsio/Makefile +++ b/drivers/crypto/chelsio/Makefile @@ -3,3 +3,4 @@ ccflags-y := -Idrivers/net/ethernet/chelsio/cxgb4 obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chcr.o chcr-objs := chcr_core.o chcr_algo.o chcr-$(CONFIG_CHELSIO_IPSEC_INLINE) += chcr_ipsec.o +obj-$(CONFIG_CRYPTO_DEV_CHELSIO_TLS) += chtls/ -- 1.8.3.1
RE: [crypto 6/8] chtls: TCB and Key program
-Original Message-
From: Stephan Mueller [mailto:smuel...@chronox.de]
Sent: Tuesday, December 5, 2017 6:37 PM
To: Atul Gupta
Cc: herb...@gondor.apana.org.au; linux-crypto@vger.kernel.org;
net...@vger.kernel.org; da...@davemloft.net; davejwat...@fb.com; Ganesh GR
; Harsh Jain
Subject: Re: [crypto 6/8] chtls: TCB and Key program
Am Dienstag, 5. Dezember 2017, 12:40:29 CET schrieb Atul Gupta:
Hi Atul,
> program the tx and rx key on chip.
>
> Signed-off-by: Atul Gupta <mailto:atul.gu...@chelsio.com>
> ---
> drivers/crypto/chelsio/chtls/chtls_hw.c | 394
> 1 file changed, 394 insertions(+)
> create mode 100644 drivers/crypto/chelsio/chtls/chtls_hw.c
>
> diff --git a/drivers/crypto/chelsio/chtls/chtls_hw.c
> b/drivers/crypto/chelsio/chtls/chtls_hw.c new file mode 100644 index
> 000..5e65aa2
> --- /dev/null
> +++ b/drivers/crypto/chelsio/chtls/chtls_hw.c
> @@ -0,0 +1,394 @@
> +/*
> + * Copyright (c) 2017 Chelsio Communications, Inc.
> + *
> + * This program is free software; you can redistribute it and/or
> +modify
> + * it under the terms of the GNU General Public License version 2 as
> + * published by the Free Software Foundation.
> + *
> + * Written by: Atul Gupta (mailto:atul.gu...@chelsio.com) */
> +
> +#include
> +#include
> +#include
> +#include
> +#include
> +#include
> +#include
> +#include
> +#include
> +#include
> +#include
> +
> +#include "chtls.h"
> +#include "chtls_cm.h"
> +
> +static void __set_tcb_field_direct(struct chtls_sock *csk,
> +struct cpl_set_tcb_field *req, u16 word,
> +u64 mask, u64 val, u8 cookie, int no_reply) {
> + struct ulptx_idata *sc;
> +
> + INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, csk->tid);
> + req->wr.wr_mid |= htonl(FW_WR_FLOWID_V(csk->tid));
> + req->reply_ctrl = htons(NO_REPLY_V(no_reply) |
> + QUEUENO_V(csk->rss_qid));
> + req->word_cookie = htons(TCB_WORD(word) | TCB_COOKIE_V(cookie));
> + req->mask = cpu_to_be64(mask);
> + req->val = cpu_to_be64(val);
> + sc = (struct ulptx_idata *)(req + 1);
> + sc->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP));
> + sc->len = htonl(0);
> +}
> +
> +void __set_tcb_field(struct sock *sk, struct sk_buff *skb, u16 word,
> + u64 mask, u64 val, u8 cookie, int no_reply) {
> + struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
> + struct cpl_set_tcb_field *req;
> + struct ulptx_idata *sc;
> + unsigned int wrlen = roundup(sizeof(*req) + sizeof(*sc), 16);
> +
> + req = (struct cpl_set_tcb_field *)__skb_put(skb, wrlen);
> + __set_tcb_field_direct(csk, req, word, mask, val, cookie, no_reply);
> + set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id); }
> +
> +static int chtls_set_tcb_field(struct sock *sk, u16 word, u64 mask,
> +u64
> val) +{
> + struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
> + struct sk_buff *skb;
> + struct cpl_set_tcb_field *req;
> + struct ulptx_idata *sc;
> + unsigned int wrlen = roundup(sizeof(*req) + sizeof(*sc), 16);
> + unsigned int credits_needed = DIV_ROUND_UP(wrlen, 16);
> +
> + skb = alloc_skb(wrlen, GFP_ATOMIC);
> + if (!skb)
> + return -ENOMEM;
> +
> + __set_tcb_field(sk, skb, word, mask, val, 0, 1);
> + set_queue(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA, sk);
> + csk->wr_credits -= credits_needed;
> + csk->wr_unacked += credits_needed;
> + enqueue_wr(csk, skb);
> + cxgb4_ofld_send(csk->egress_dev, skb);
> + return 0;
> +}
> +
> +/*
> + * Set one of the t_flags bits in the TCB.
> + */
> +int chtls_set_tcb_tflag(struct sock *sk, unsigned int bit_pos, int
> +val) {
> + return chtls_set_tcb_field(sk, 1, 1ULL << bit_pos,
> + val << bit_pos);
> +}
> +
> +static int chtls_set_tcb_keyid(struct sock *sk, int keyid) {
> + return chtls_set_tcb_field(sk, 31, 0xULL, keyid); }
> +
> +static int chtls_set_tcb_seqno(struct sock *sk) {
> + return chtls_set_tcb_field(sk, 28, ~0ULL, 0); }
> +
> +static int chtls_set_tcb_quiesce(struct sock *sk, int val) {
> + return chtls_set_tcb_field(sk, 1, (1ULL << TF_RX_QUIESCE_S),
> +TF_RX_QUIESCE_V(val));
> +}
> +
> +static void *chtls_alloc_mem(unsigned long size) {
> + void *p = kmalloc(size, GFP_KERNEL);
> +
> + if (!p)
> + p = vmalloc(size);
> + if (p)
> + memset(p, 0, size);
> +
RE: [crypto 4/8] chtls: CPL handler definition
-Original Message-
From: linux-crypto-ow...@vger.kernel.org
[mailto:linux-crypto-ow...@vger.kernel.org] On Behalf Of Stefano Brivio
Sent: Tuesday, December 5, 2017 8:54 PM
To: Atul Gupta
Cc: herb...@gondor.apana.org.au; linux-crypto@vger.kernel.org;
net...@vger.kernel.org; da...@davemloft.net; davejwat...@fb.com; Ganesh GR
; Harsh Jain
Subject: Re: [crypto 4/8] chtls: CPL handler definition
On Tue, 5 Dec 2017 17:10:00 +0530
Atul Gupta wrote:
> CPL handlers for TLS session, record transmit and receive
>
> Signed-off-by: Atul Gupta
> ---
> drivers/crypto/chelsio/chtls/chtls_cm.c | 2048
> +++
> 1 file changed, 2048 insertions(+)
> create mode 100644 drivers/crypto/chelsio/chtls/chtls_cm.c
>
> diff --git a/drivers/crypto/chelsio/chtls/chtls_cm.c
> b/drivers/crypto/chelsio/chtls/chtls_cm.c
> new file mode 100644
> index 000..ea1c301
> --- /dev/null
> +++ b/drivers/crypto/chelsio/chtls/chtls_cm.c
> @@ -0,0 +1,2048 @@
> +/*
> + * Copyright (c) 2017 Chelsio Communications, Inc.
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License version 2 as
> + * published by the Free Software Foundation.
> + *
> + * Written by: Atul Gupta (atul.gu...@chelsio.com)
> + */
> +
> +#include
> +#include
> +#include
> +#include
> +#include
> +#include
> +#include
> +#include
> +#include
> +#include
> +#include
> +#include
> +#include
> +#include
> +#include
> +
> +#include "chtls.h"
> +#include "chtls_cm.h"
> +
> +extern struct request_sock_ops chtls_rsk_ops;
> +static void (*tcp_time_wait_p)(struct sock *sk, int state, int timeo);
> +
> +/*
> + * State transitions and actions for close. Note that if we are in SYN_SENT
> + * we remain in that state as we cannot control a connection while it's in
> + * SYN_SENT; such connections are allowed to establish and are then aborted.
> + */
> +static unsigned char new_state[16] = {
> + /* current state: new state: action: */
> + /* (Invalid) */ TCP_CLOSE,
> + /* TCP_ESTABLISHED */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
> + /* TCP_SYN_SENT*/ TCP_SYN_SENT,
> + /* TCP_SYN_RECV*/ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
> + /* TCP_FIN_WAIT1 */ TCP_FIN_WAIT1,
> + /* TCP_FIN_WAIT2 */ TCP_FIN_WAIT2,
> + /* TCP_TIME_WAIT */ TCP_CLOSE,
> + /* TCP_CLOSE */ TCP_CLOSE,
> + /* TCP_CLOSE_WAIT */ TCP_LAST_ACK | TCP_ACTION_FIN,
> + /* TCP_LAST_ACK*/ TCP_LAST_ACK,
> + /* TCP_LISTEN */ TCP_CLOSE,
> + /* TCP_CLOSING */ TCP_CLOSING,
> +};
> +
> +static struct chtls_sock *chtls_sock_create(struct chtls_dev *cdev)
> +{
> + struct chtls_sock *csk = kzalloc(sizeof(*csk), GFP_NOIO);
> +
> + if (!csk)
> + return NULL;
> +
> + csk->txdata_skb_cache = alloc_skb(TXDATA_SKB_LEN, GFP_ATOMIC);
Excess whitespace.
[Atul] Will take care in v2
> + if (!csk->txdata_skb_cache) {
> + kfree(csk);
> + return NULL;
> + }
> +
> + kref_init(&csk->kref);
> + csk->cdev = cdev;
> + skb_queue_head_init(&csk->txq);
> + csk->wr_skb_head = NULL;
> + csk->wr_skb_tail = NULL;
> + csk->mss = MAX_MSS;
> + csk->tlshws.ofld = 1;
> + csk->tlshws.txkey = -1;
> + csk->tlshws.rxkey = -1;
> + csk->tlshws.mfs = TLS_MFS;
> + skb_queue_head_init(&csk->tlshws.sk_recv_queue);
> + return csk;
> +}
> +
> +void chtls_sock_release(struct kref *ref)
> +{
> + struct chtls_sock *csk =
> + container_of(ref, struct chtls_sock, kref);
> +
> + kfree(csk);
> +}
> +
> +void get_tcp_symbol(void)
> +{
> + tcp_time_wait_p = (void *)kallsyms_lookup_name("tcp_time_wait");
> + if (!tcp_time_wait_p)
> + pr_info("could not locate tcp_time_wait");
Probably not something that should be used here. Why do you need this?
[Atul] using it to call tcp_time_wait, as used in tcp_rcv_state_process
> +}
> +
> +static struct net_device *chtls_ipv4_netdev(struct chtls_dev *cdev,
> + struct sock *sk)
> +{
> + struct net_device *ndev = cdev->ports[0];
> +
> + if (likely(!inet_sk(sk)->inet_rcv_saddr))
> + return ndev;
> +
> + ndev = ip_dev_find(&init_net, inet_sk(sk)->inet_rcv_saddr);
> + if (!ndev)
> + return NULL;
> +
> + if (is_vlan_dev(ndev))
> + return vlan_dev_real_dev(ndev);
> + return ndev;
> +}
> +
> +sta
RE: [crypto 6/8] chtls: TCB and Key program
-Original Message- From: linux-crypto-ow...@vger.kernel.org [mailto:linux-crypto-ow...@vger.kernel.org] On Behalf Of Stephan Mueller Sent: Thursday, December 7, 2017 8:13 PM To: Atul Gupta Cc: herb...@gondor.apana.org.au; linux-crypto@vger.kernel.org; net...@vger.kernel.org; da...@davemloft.net; davejwat...@fb.com; Ganesh GR ; Harsh Jain Subject: Re: [crypto 6/8] chtls: TCB and Key program Am Donnerstag, 7. Dezember 2017, 15:21:03 CET schrieb Atul Gupta: Hi Atul, > > memzero_explicit(key)? > [Atul] may not be required as entire info of size keylen and > AEAD_H_SIZE is copied onto kctx->key. Key data is received from user, > while ghash is memset and locally generated Sure, but wouldn't it make sense to zap all instances where key material was stored? Agree, Its safe to memset where keylen is variable, perhaps in future where we support different keylen. In current case key len is same as buffer size hence may not cause issue. > > As far as I see, the key is part of the skb (via kctx). This skb is > released after being processed. The release calls kfree_skb which does > not zeroize the key. Wouldn't it make sense to clear the memory of the > key when the skb is released? [Atul] we should perhaps memset the info > received from user so that driver has no info on key once its written on chip > memory. > memset(gcm_ctx->key, 0, keylen); Are you saying that the skb (via kctx) above does not obtain a copy of the key? If not, what is done in chtls_key_info? It does have a key copy, I was not sure how key info is accessed once skb is released. Ciao Stephan Thanks Atul
[RFC crypto v2 0/9] Chelsio Inline TLS
RFC series for Chelsio Inline TLS driver (chtls.ko)
Driver use the ULP infrastructure to register chtls as Inline TLS ULP.
Chtls use TCP Sockets to transmit and receive TLS record. TCP proto_ops
is extended to offload TLS record.
T6 adapter provides the following features:
-TLS record offload, TLS header, encrypt, digest and transmit
-TLS record receive and decrypt
-TLS keys store
-TCP/IP engine
-TLS engine
-GCM crypto engine [support CBC also]
TLS provides security at the transport layer. It uses TCP to provide
reliable end-to-end transport of application data. It relies on TCP
for any retransmission. TLS session comprises of three parts:
a. TCP/IP connection
b. TLS handshake
c. Record layer processing
TLS handshake state machine is executed in host (refer standard
implementation eg. OpenSSL). Setsockopt [SOL_TCP, TCP_ULP] initialize
TCP proto-ops for Chelsio inline tls support. setsockopt(sock, SOL_TCP,
TCP_ULP, "chtls", sizeof("chtls"));
Tx and Rx Keys are decided during handshake and programmed onto the chip
after CCS is exchanged.
struct tls12_crypto_info_aes_gcm_128 crypto_info
setsockopt(sock, SOL_TLS, TLS_TX, &crypto_info, sizeof(crypto_info))
Finish is the first encrypted/decrypted message tx/rx inline.
On the Tx path TLS engine receive plain text from openssl, insert IV,
fetches the tx key, create cipher text records and generate MAC. TLS
header is added to cipher text and forward to TCP/IP engine for transport
layer processing and transmission on wire.
TX:
Application--openssl--chtls---TLS engine---encrypt/auth---TCP/IP
engine---wire.
On the Rx side, data received is PDU aligned at record
boundaries. TLS processes only the complete record. If rx key is programmed
on CCS receive, data is decrypted and plain text is posted to host.
RX:
Wire--cipher-text--TCP/IP engine [PDU align]---TLS engine---
decrypt/auth---plain-text--chtls--openssl--application
v2: fixed the following based on the review comments of Stephan Mueller,
Stefano Brivio and Hannes Frederic
-Added more details in cover letter
-Fixed indentation and formating issues
-Using aes instead of aes-generic
-memset key info after programing the key on chip
-reordered the patch sequence
Atul Gupta (9):
chtls: structure and macro definiton
cxgb4: Inline TLS FW Interface
cxgb4: LLD driver changes to enable TLS
chcr: Key Macro
chtls: Key program
chtls: CPL handler definition
chtls: Inline crypto request Tx/Rx
chtls: Register the ULP
Makefile Kconfig
drivers/crypto/chelsio/Kconfig | 10 +
drivers/crypto/chelsio/Makefile|1 +
drivers/crypto/chelsio/chcr_algo.h | 42 +
drivers/crypto/chelsio/chcr_core.h | 55 +-
drivers/crypto/chelsio/chtls/Makefile |4 +
drivers/crypto/chelsio/chtls/chtls.h | 480 +
drivers/crypto/chelsio/chtls/chtls_cm.c| 2045
drivers/crypto/chelsio/chtls/chtls_cm.h| 203 ++
drivers/crypto/chelsio/chtls/chtls_hw.c| 394
drivers/crypto/chelsio/chtls/chtls_io.c| 1866 ++
drivers/crypto/chelsio/chtls/chtls_main.c | 584 ++
drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c | 18 +-
drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c| 32 +-
drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h |7 +
drivers/net/ethernet/chelsio/cxgb4/sge.c | 98 +-
drivers/net/ethernet/chelsio/cxgb4/t4_msg.h| 121 +-
drivers/net/ethernet/chelsio/cxgb4/t4_regs.h |2 +
drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h | 165 +-
include/uapi/linux/tls.h |1 +
net/ipv4/tcp_minisocks.c |1 +
20 files changed, 6110 insertions(+), 19 deletions(-)
create mode 100644 drivers/crypto/chelsio/chtls/Makefile
create mode 100644 drivers/crypto/chelsio/chtls/chtls.h
create mode 100644 drivers/crypto/chelsio/chtls/chtls_cm.c
create mode 100644 drivers/crypto/chelsio/chtls/chtls_cm.h
create mode 100644 drivers/crypto/chelsio/chtls/chtls_hw.c
create mode 100644 drivers/crypto/chelsio/chtls/chtls_io.c
create mode 100644 drivers/crypto/chelsio/chtls/chtls_main.c
--
1.8.3.1
[RFC crypto v2 1/9] chtls: structure and macro definiton
Inline TLS state, connection management. Supporting macros definition.
Signed-off-by: Atul Gupta
---
drivers/crypto/chelsio/chtls/chtls.h| 480
drivers/crypto/chelsio/chtls/chtls_cm.h | 203 ++
2 files changed, 683 insertions(+)
create mode 100644 drivers/crypto/chelsio/chtls/chtls.h
create mode 100644 drivers/crypto/chelsio/chtls/chtls_cm.h
diff --git a/drivers/crypto/chelsio/chtls/chtls.h
b/drivers/crypto/chelsio/chtls/chtls.h
new file mode 100644
index 000..33d3ed4
--- /dev/null
+++ b/drivers/crypto/chelsio/chtls/chtls.h
@@ -0,0 +1,480 @@
+/*
+ * Copyright (c) 2016 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef __CHTLS_H__
+#define __CHTLS_H__
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "t4fw_api.h"
+#include "t4_msg.h"
+#include "cxgb4.h"
+#include "cxgb4_uld.h"
+#include "l2t.h"
+#include "chcr_algo.h"
+#include "chcr_core.h"
+#include "chcr_crypto.h"
+
+#define CIPHER_BLOCK_SIZE 16
+#define MAX_IVS_PAGE256
+#define TLS_KEY_CONTEXT_SZ 64
+#define TLS_HEADER_LENGTH 5
+#define SCMD_CIPH_MODE_AES_GCM 2
+#define GCM_TAG_SIZE16
+#define AEAD_EXPLICIT_DATA_SIZE 8
+/* Any MFS size should work and come from openssl */
+#define TLS_MFS16384
+
+#define SOCK_INLINE (31)
+#define RSS_HDR sizeof(struct rss_header)
+
+enum {
+ CHTLS_KEY_CONTEXT_DSGL,
+ CHTLS_KEY_CONTEXT_IMM,
+ CHTLS_KEY_CONTEXT_DDR,
+};
+
+enum {
+ CHTLS_LISTEN_START,
+ CHTLS_LISTEN_STOP,
+};
+
+/* Flags for return value of CPL message handlers */
+enum {
+ CPL_RET_BUF_DONE = 1, /* buffer processing done */
+ CPL_RET_BAD_MSG = 2,/* bad CPL message */
+ CPL_RET_UNKNOWN_TID = 4 /* unexpected unknown TID */
+};
+
+#define TLS_RCV_ST_READ_HEADER 0xF0
+#define TLS_RCV_ST_READ_BODY0xF1
+#define TLS_RCV_ST_READ_DONE0xF2
+#define TLS_RCV_ST_READ_NB 0xF3
+
+#define RSPQ_HASH_BITS 5
+#define LISTEN_INFO_HASH_SIZE 32
+struct listen_info {
+ struct listen_info *next; /* Link to next entry */
+ struct sock *sk; /* The listening socket */
+ unsigned int stid; /* The server TID */
+};
+
+enum {
+ T4_LISTEN_START_PENDING,
+ T4_LISTEN_STARTED
+};
+
+enum csk_flags {
+ CSK_CALLBACKS_CHKD, /* socket callbacks have been sanitized */
+ CSK_ABORT_REQ_RCVD, /* received one ABORT_REQ_RSS message */
+ CSK_TX_MORE_DATA, /* sending ULP data; don't set SHOVE bit */
+ CSK_TX_WAIT_IDLE, /* suspend Tx until in-flight data is ACKed */
+ CSK_ABORT_SHUTDOWN, /* shouldn't send more abort requests */
+ CSK_ABORT_RPL_PENDING, /* expecting an abort reply */
+ CSK_CLOSE_CON_REQUESTED,/* we've sent a close_conn_req */
+ CSK_TX_DATA_SENT, /* sent a TX_DATA WR on this connection */
+ CSK_TX_FAILOVER,/* Tx traffic failing over */
+ CSK_UPDATE_RCV_WND, /* Need to update rcv window */
+ CSK_RST_ABORTED,/* outgoing RST was aborted */
+ CSK_TLS_HANDSHK,/* TLS Handshake */
+};
+
+struct listen_ctx {
+ struct sock *lsk;
+ struct chtls_dev *cdev;
+ u32 state;
+};
+
+struct key_map {
+ unsigned long *addr;
+ unsigned int start;
+ unsigned int available;
+ unsigned int size;
+ spinlock_t lock; /* lock for key id request from map */
+} __packed;
+
+struct tls_scmd {
+ __be32 seqno_numivs;
+ __be32 ivgen_hdrlen;
+};
+
+struct chtls_dev {
+ struct list_head list;
+ struct cxgb4_lld_info *lldi;
+ struct pci_dev *pdev;
+ struct listen_info *listen_hash_tab[LISTEN_INFO_HASH_SIZE];
+ spinlock_t listen_lock; /* lock for listen list */
+ struct net_device **ports;
+ struct tid_info *tids;
+ unsigned int pfvf;
+ const unsigned short *mtus;
+
+ spinlock_t aidr_lock cacheline_aligned_in_smp;
+ struct idr aidr; /* ATID id space */
+ struct idr hwtid_idr;
+ struct idr stid_idr;
+
+ spinlock_t idr_lock cacheline_aligned_in_smp;
+
+ struct net_device *egr_dev[NCHAN * 2];
+ struct sk_buff *rspq_skb_cache[1 << RSPQ_HASH_BITS];
+ struct sk_buff *askb;
+
+ struct sk_buff_head deferq;
+ struct work_struct deferq_task;
+
+ struct list_head list_node;
+ struct list_head rcu_node;
+ struct list_head na_node;
+ unsigned int send_page_order;
+
[RFC crypto v2 2/9] cxgb4: Inline TLS FW Interface
Key area size in hw-config file. CPL struct for TLS request
and response. Work request for Inline TLS.
Signed-off-by: Atul Gupta
---
drivers/net/ethernet/chelsio/cxgb4/t4_msg.h | 121 ++-
drivers/net/ethernet/chelsio/cxgb4/t4_regs.h | 2 +
drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h | 165 +-
3 files changed, 283 insertions(+), 5 deletions(-)
diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h
b/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h
index 7e12f24..9a56e0d 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h
@@ -81,6 +81,7 @@ enum {
CPL_RX_ISCSI_CMP = 0x45,
CPL_TRACE_PKT_T5 = 0x48,
CPL_RX_ISCSI_DDP = 0x49,
+ CPL_RX_TLS_CMP= 0x4E,
CPL_RDMA_READ_REQ = 0x60,
@@ -88,6 +89,7 @@ enum {
CPL_ACT_OPEN_REQ6 = 0x83,
CPL_TX_TLS_PDU =0x88,
+ CPL_TX_TLS_SFO= 0x89,
CPL_TX_SEC_PDU= 0x8A,
CPL_TX_TLS_ACK= 0x8B,
@@ -97,6 +99,7 @@ enum {
CPL_RX_MPS_PKT= 0xAF,
CPL_TRACE_PKT = 0xB0,
+ CPL_TLS_DATA = 0xB1,
CPL_ISCSI_DATA= 0xB2,
CPL_FW4_MSG = 0xC0,
@@ -150,6 +153,7 @@ enum {
ULP_MODE_RDMA = 4,
ULP_MODE_TCPDDP= 5,
ULP_MODE_FCOE = 6,
+ ULP_MODE_TLS = 8,
};
enum {
@@ -1414,6 +1418,14 @@ struct cpl_tx_data {
#define TX_FORCE_S 13
#define TX_FORCE_V(x) ((x) << TX_FORCE_S)
+#define TX_SHOVE_S14
+#define TX_SHOVE_V(x) ((x) << TX_SHOVE_S)
+
+#define TX_ULP_MODE_S10
+#define TX_ULP_MODE_M0x7
+#define TX_ULP_MODE_V(x) ((x) << TX_ULP_MODE_S)
+#define TX_ULP_MODE_G(x) (((x) >> TX_ULP_MODE_S) & TX_ULP_MODE_M)
+
#define T6_TX_FORCE_S 20
#define T6_TX_FORCE_V(x) ((x) << T6_TX_FORCE_S)
#define T6_TX_FORCE_F T6_TX_FORCE_V(1U)
@@ -1428,12 +1440,21 @@ enum {
ULP_TX_SC_NOOP = 0x80,
ULP_TX_SC_IMM = 0x81,
ULP_TX_SC_DSGL = 0x82,
- ULP_TX_SC_ISGL = 0x83
+ ULP_TX_SC_ISGL = 0x83,
+ ULP_TX_SC_MEMRD = 0x86
};
#define ULPTX_CMD_S24
#define ULPTX_CMD_V(x) ((x) << ULPTX_CMD_S)
+#define ULPTX_LEN16_S0
+#define ULPTX_LEN16_M0xFF
+#define ULPTX_LEN16_V(x) ((x) << ULPTX_LEN16_S)
+
+#define ULP_TX_SC_MORE_S 23
+#define ULP_TX_SC_MORE_V(x) ((x) << ULP_TX_SC_MORE_S)
+#define ULP_TX_SC_MORE_F ULP_TX_SC_MORE_V(1U)
+
struct ulptx_sge_pair {
__be32 len[2];
__be64 addr[2];
@@ -1948,4 +1969,102 @@ enum {
X_CPL_RX_MPS_PKT_TYPE_QFC = 1 << 2,
X_CPL_RX_MPS_PKT_TYPE_PTP = 1 << 3
};
+
+struct cpl_tx_tls_sfo {
+ __be32 op_to_seg_len;
+ __be32 pld_len;
+ __be32 type_protover;
+ __be32 r1_lo;
+ __be32 seqno_numivs;
+ __be32 ivgen_hdrlen;
+ __be64 scmd1;
+};
+
+/* cpl_tx_tls_sfo macros */
+#define CPL_TX_TLS_SFO_OPCODE_S 24
+#define CPL_TX_TLS_SFO_OPCODE_V(x) ((x) << CPL_TX_TLS_SFO_OPCODE_S)
+
+#define CPL_TX_TLS_SFO_DATA_TYPE_S 20
+#define CPL_TX_TLS_SFO_DATA_TYPE_V(x) ((x) << CPL_TX_TLS_SFO_DATA_TYPE_S)
+
+#define CPL_TX_TLS_SFO_CPL_LEN_S16
+#define CPL_TX_TLS_SFO_CPL_LEN_V(x) ((x) << CPL_TX_TLS_SFO_CPL_LEN_S)
+
+#define CPL_TX_TLS_SFO_SEG_LEN_S0
+#define CPL_TX_TLS_SFO_SEG_LEN_M0x
+#define CPL_TX_TLS_SFO_SEG_LEN_V(x) ((x) << CPL_TX_TLS_SFO_SEG_LEN_S)
+#define CPL_TX_TLS_SFO_SEG_LEN_G(x) \
+ (((x) >> CPL_TX_TLS_SFO_SEG_LEN_S) & CPL_TX_TLS_SFO_SEG_LEN_M)
+
+#define CPL_TX_TLS_SFO_TYPE_S 24
+#define CPL_TX_TLS_SFO_TYPE_M 0xff
+#define CPL_TX_TLS_SFO_TYPE_V(x)((x) << CPL_TX_TLS_SFO_TYPE_S)
+#define CPL_TX_TLS_SFO_TYPE_G(x)\
+ (((x) >> CPL_TX_TLS_SFO_TYPE_S) & CPL_TX_TLS_SFO_TYPE_M)
+
+#define CPL_TX_TLS_SFO_PROTOVER_S 8
+#define CPL_TX_TLS_SFO_PROTOVER_M 0x
+#define CPL_TX_TLS_SFO_PROTOVER_V(x)((x) << CPL_TX_TLS_SFO_PROTOVER_S)
+#define CPL_TX_TLS_SFO_PROTOVER_G(x)\
+ (((x) >> CPL_TX_TLS_SFO_PROTOVER_S) & CPL_TX_TLS_SFO_PROTOVER_M)
+
+struct cpl_tls_data {
+ struct rss_header rsshdr;
+ union opcode_tid ot;
+ __be32 length_pkd;
+ __be32 seq;
+ __be32 r1;
+};
+
+#define CPL_TLS_DATA_OPCODE_S 24
+#define CPL_TLS_DATA_OPCODE_M 0xff
+#define CPL_TLS_DATA_OPCODE_V(x)((x) << CPL_TLS_DATA_OPCODE_S)
+#define CPL_TLS_DATA_OPCODE_G(x)\
+ (((x) >> CPL_TLS_DATA_OPCODE_S) & CPL_TLS_DATA_OPCODE_M)
+
+#define CPL_TLS_DATA_TID_S 0
+#define CPL_TLS_DATA_TID_M 0xff
+#define CPL_TLS_DATA_TID_V(x) ((x) << CPL_TLS_DATA_TID_S)
+#define CPL_TLS_DATA_TID_G(x) \
+ (((x) >> CPL_TLS_DATA_TID_S) &
[RFC crypto v2 4/9] chcr: Key Macro
Define macro for TLS Key context
Signed-off-by: Atul Gupta
---
drivers/crypto/chelsio/chcr_algo.h | 42 +
drivers/crypto/chelsio/chcr_core.h | 55 +-
2 files changed, 96 insertions(+), 1 deletion(-)
diff --git a/drivers/crypto/chelsio/chcr_algo.h
b/drivers/crypto/chelsio/chcr_algo.h
index d1673a5..f263cd4 100644
--- a/drivers/crypto/chelsio/chcr_algo.h
+++ b/drivers/crypto/chelsio/chcr_algo.h
@@ -86,6 +86,39 @@
KEY_CONTEXT_OPAD_PRESENT_M)
#define KEY_CONTEXT_OPAD_PRESENT_F KEY_CONTEXT_OPAD_PRESENT_V(1U)
+#define TLS_KEYCTX_RXFLIT_CNT_S 24
+#define TLS_KEYCTX_RXFLIT_CNT_V(x) ((x) << TLS_KEYCTX_RXFLIT_CNT_S)
+
+#define TLS_KEYCTX_RXPROT_VER_S 20
+#define TLS_KEYCTX_RXPROT_VER_M 0xf
+#define TLS_KEYCTX_RXPROT_VER_V(x) ((x) << TLS_KEYCTX_RXPROT_VER_S)
+
+#define TLS_KEYCTX_RXCIPH_MODE_S 16
+#define TLS_KEYCTX_RXCIPH_MODE_M 0xf
+#define TLS_KEYCTX_RXCIPH_MODE_V(x) ((x) << TLS_KEYCTX_RXCIPH_MODE_S)
+
+#define TLS_KEYCTX_RXAUTH_MODE_S 12
+#define TLS_KEYCTX_RXAUTH_MODE_M 0xf
+#define TLS_KEYCTX_RXAUTH_MODE_V(x) ((x) << TLS_KEYCTX_RXAUTH_MODE_S)
+
+#define TLS_KEYCTX_RXCIAU_CTRL_S 11
+#define TLS_KEYCTX_RXCIAU_CTRL_V(x) ((x) << TLS_KEYCTX_RXCIAU_CTRL_S)
+
+#define TLS_KEYCTX_RX_SEQCTR_S 9
+#define TLS_KEYCTX_RX_SEQCTR_M 0x3
+#define TLS_KEYCTX_RX_SEQCTR_V(x) ((x) << TLS_KEYCTX_RX_SEQCTR_S)
+
+#define TLS_KEYCTX_RX_VALID_S 8
+#define TLS_KEYCTX_RX_VALID_V(x) ((x) << TLS_KEYCTX_RX_VALID_S)
+
+#define TLS_KEYCTX_RXCK_SIZE_S 3
+#define TLS_KEYCTX_RXCK_SIZE_M 0x7
+#define TLS_KEYCTX_RXCK_SIZE_V(x) ((x) << TLS_KEYCTX_RXCK_SIZE_S)
+
+#define TLS_KEYCTX_RXMK_SIZE_S 0
+#define TLS_KEYCTX_RXMK_SIZE_M 0x7
+#define TLS_KEYCTX_RXMK_SIZE_V(x) ((x) << TLS_KEYCTX_RXMK_SIZE_S)
+
#define CHCR_HASH_MAX_DIGEST_SIZE 64
#define CHCR_MAX_SHA_DIGEST_SIZE 64
@@ -176,6 +209,15 @@
KEY_CONTEXT_SALT_PRESENT_V(1) | \
KEY_CONTEXT_CTX_LEN_V((ctx_len)))
+#define FILL_KEY_CRX_HDR(ck_size, mk_size, d_ck, opad, ctx_len) \
+ htonl(TLS_KEYCTX_RXMK_SIZE_V(mk_size) | \
+ TLS_KEYCTX_RXCK_SIZE_V(ck_size) | \
+ TLS_KEYCTX_RX_VALID_V(1) | \
+ TLS_KEYCTX_RX_SEQCTR_V(3) | \
+ TLS_KEYCTX_RXAUTH_MODE_V(4) | \
+ TLS_KEYCTX_RXCIPH_MODE_V(2) | \
+ TLS_KEYCTX_RXFLIT_CNT_V((ctx_len)))
+
#define FILL_WR_OP_CCTX_SIZE \
htonl( \
FW_CRYPTO_LOOKASIDE_WR_OPCODE_V( \
diff --git a/drivers/crypto/chelsio/chcr_core.h
b/drivers/crypto/chelsio/chcr_core.h
index 3c29ee0..77056a9 100644
--- a/drivers/crypto/chelsio/chcr_core.h
+++ b/drivers/crypto/chelsio/chcr_core.h
@@ -65,10 +65,58 @@
struct _key_ctx {
__be32 ctx_hdr;
u8 salt[MAX_SALT];
- __be64 reserverd;
+ __be64 iv_to_auth;
unsigned char key[0];
};
+#define KEYCTX_TX_WR_IV_S 55
+#define KEYCTX_TX_WR_IV_M 0x1ffULL
+#define KEYCTX_TX_WR_IV_V(x) ((x) << KEYCTX_TX_WR_IV_S)
+#define KEYCTX_TX_WR_IV_G(x) \
+ (((x) >> KEYCTX_TX_WR_IV_S) & KEYCTX_TX_WR_IV_M)
+
+#define KEYCTX_TX_WR_AAD_S 47
+#define KEYCTX_TX_WR_AAD_M 0xffULL
+#define KEYCTX_TX_WR_AAD_V(x) ((x) << KEYCTX_TX_WR_AAD_S)
+#define KEYCTX_TX_WR_AAD_G(x) (((x) >> KEYCTX_TX_WR_AAD_S) & \
+ KEYCTX_TX_WR_AAD_M)
+
+#define KEYCTX_TX_WR_AADST_S 39
+#define KEYCTX_TX_WR_AADST_M 0xffULL
+#define KEYCTX_TX_WR_AADST_V(x) ((x) << KEYCTX_TX_WR_AADST_S)
+#define KEYCTX_TX_WR_AADST_G(x) \
+ (((x) >> KEYCTX_TX_WR_AADST_S) & KEYCTX_TX_WR_AADST_M)
+
+#define KEYCTX_TX_WR_CIPHER_S 30
+#define KEYCTX_TX_WR_CIPHER_M 0x1ffULL
+#define KEYCTX_TX_WR_CIPHER_V(x) ((x) << KEYCTX_TX_WR_CIPHER_S)
+#define KEYCTX_TX_WR_CIPHER_G(x) \
+ (((x) >> KEYCTX_TX_WR_CIPHER_S) & KEYCTX_TX_WR_CIPHER_M)
+
+#define KEYCTX_TX_WR_CIPHERST_S 23
+#define KEYCTX_TX_WR_CIPHERST_M 0x7f
+#define KEYCTX_TX_WR_CIPHERST_V(x) ((x) << KEYCTX_TX_WR_CIPHERST_S)
+#define KEYCTX_TX_WR_CIPHERST_G(x) \
+ (((x) >> KEYCTX_TX_WR_CIPHERST_S) & KEYCTX_TX_WR_CIPHERST_M)
+
+#define KEYCTX_TX_WR_AUTH_S 14
+#define KEYCTX_TX_WR_AUTH_M 0x1ff
+#define KEYCTX_TX_WR_AUTH_V(x) ((x) << KEYCTX_TX_WR_AUTH_S)
+#define KEYCTX_TX_WR_AUTH_G(x) \
+ (((x) >> KEYCTX_TX_WR_AUTH_S) & KEYCTX_TX_WR_AUTH_M)
+
+#define KEYCTX_TX_WR_AUTHST_S 7
+#define KEYCTX_TX_WR_AUTHST_M 0x7f
+#define KEYCTX_TX_WR_AUTHST_V(x) ((x) << KEYCTX_TX_WR_AUTHST_S)
+#define KEYCTX_TX_WR_AUTHST_G(x) \
+ (((x) >> KEYCTX_TX_WR_AUTHST_S) & KEYCTX_TX_WR_AUTHST_M)
+
+#define KEYCTX_TX_WR_AUTHIN_S 0
+#define KEYCTX_TX_WR_AUTHIN_M 0x7f
+#define KEYCTX_TX_WR_AUTHIN_V(x) ((x) << KEYCTX_TX_WR_AUTHIN_S)
+#define KEYCTX_TX_WR_AUTHIN_G(x) \
+ (((x) >> KEYCTX_TX_WR_AUTHIN_S) & KEYCTX_TX_WR_AUTH
[RFC crypto v2 3/9] cxgb4: LLD driver changes to enable TLS
Read FW capability. Read key area size. Dump the TLS record count.
Signed-off-by: Atul Gupta
---
drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c | 18 +++-
drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c| 32 +--
drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h | 7 ++
drivers/net/ethernet/chelsio/cxgb4/sge.c | 98 +-
4 files changed, 142 insertions(+), 13 deletions(-)
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
index cf47183..cfc9210 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
@@ -2826,8 +2826,8 @@ static int meminfo_show(struct seq_file *seq, void *v)
"Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
"TDDP region:", "TPT region:", "STAG region:", "RQ region:",
"RQUDP region:", "PBL region:", "TXPBL region:",
- "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
- "On-chip queues:"
+ "TLSKey region:", "DBVFIFO region:", "ULPRX state:",
+ "ULPTX state:", "On-chip queues:"
};
int i, n;
@@ -2943,6 +2943,12 @@ static int meminfo_show(struct seq_file *seq, void *v)
ulp_region(RX_RQUDP);
ulp_region(RX_PBL);
ulp_region(TX_PBL);
+ if (adap->params.crypto & FW_CAPS_CONFIG_TLS_INLINE) {
+ ulp_region(RX_TLS_KEY);
+ } else {
+ md->base = 0;
+ md->idx = ARRAY_SIZE(region);
+ }
#undef ulp_region
md->base = 0;
md->idx = ARRAY_SIZE(region);
@@ -3098,6 +3104,14 @@ static int chcr_show(struct seq_file *seq, void *v)
atomic_read(&adap->chcr_stats.fallback));
seq_printf(seq, "IPSec PDU: %10u\n",
atomic_read(&adap->chcr_stats.ipsec_cnt));
+
+ seq_puts(seq, "\nChelsio Inline TLS Stats\n");
+ seq_printf(seq, "TLS PDU Tx: %u\n",
+ atomic_read(&adap->chcr_stats.tls_pdu_tx));
+ seq_printf(seq, "TLS PDU Rx: %u\n",
+ atomic_read(&adap->chcr_stats.tls_pdu_rx));
+ seq_printf(seq, "TLS Keys (DDR) Count: %u\n",
+ atomic_read(&adap->chcr_stats.tls_key));
return 0;
}
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
index 05a4abf..60eb18b 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
@@ -4086,18 +4086,32 @@ static int adap_init0(struct adapter *adap)
adap->num_ofld_uld += 2;
}
if (caps_cmd.cryptocaps) {
- /* Should query params here...TODO */
- params[0] = FW_PARAM_PFVF(NCRYPTO_LOOKASIDE);
- ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2,
- params, val);
- if (ret < 0) {
- if (ret != -EINVAL)
+ if (ntohs(caps_cmd.cryptocaps) &
+ FW_CAPS_CONFIG_CRYPTO_LOOKASIDE) {
+ params[0] = FW_PARAM_PFVF(NCRYPTO_LOOKASIDE);
+ ret = t4_query_params(adap, adap->mbox, adap->pf, 0,
+ 2, params, val);
+ if (ret < 0) {
+ if (ret != -EINVAL)
+ goto bye;
+ } else {
+ adap->vres.ncrypto_fc = val[0];
+ }
+ adap->num_ofld_uld += 1;
+ }
+ if (ntohs(caps_cmd.cryptocaps) &
+ FW_CAPS_CONFIG_TLS_INLINE) {
+ params[0] = FW_PARAM_PFVF(TLS_START);
+ params[1] = FW_PARAM_PFVF(TLS_END);
+ ret = t4_query_params(adap, adap->mbox, adap->pf, 0,
+ 2, params, val);
+ if (ret < 0)
goto bye;
- } else {
- adap->vres.ncrypto_fc = val[0];
+ adap->vres.key.start = val[0];
+ adap->vres.key.size = val[1] - val[0] + 1;
+ adap->num_uld += 1;
}
adap->params.crypto = ntohs(caps_cmd.cryptocaps);
- adap->num_uld += 1;
}
#undef FW_PARAM_PFVF
#undef FW_PARAM_DEV
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h
b/drivers/
[RFC crypto v2 7/9] chtls: Inline crypto request Tx/Rx
TLS handler for record transmit and receive.
Create Inline TLS work request and post to FW.
Signed-off-by: Atul Gupta
---
drivers/crypto/chelsio/chtls/chtls_io.c | 1866 +++
1 file changed, 1866 insertions(+)
create mode 100644 drivers/crypto/chelsio/chtls/chtls_io.c
diff --git a/drivers/crypto/chelsio/chtls/chtls_io.c
b/drivers/crypto/chelsio/chtls/chtls_io.c
new file mode 100644
index 000..de52a64
--- /dev/null
+++ b/drivers/crypto/chelsio/chtls/chtls_io.c
@@ -0,0 +1,1866 @@
+/*
+ * Copyright (c) 2017 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Written by: Atul Gupta (atul.gu...@chelsio.com)
+ */
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "chtls.h"
+#include "chtls_cm.h"
+
+static bool is_tls_hw(struct chtls_sock *csk)
+{
+ return csk->tlshws.ofld;
+}
+
+static bool is_tls_rx(struct chtls_sock *csk)
+{
+ return (csk->tlshws.rxkey >= 0);
+}
+
+static bool is_tls_tx(struct chtls_sock *csk)
+{
+ return (csk->tlshws.txkey >= 0);
+}
+
+static bool is_tls_skb(struct chtls_sock *csk, const struct sk_buff *skb)
+{
+ return (is_tls_hw(csk) && skb_ulp_tls_skb_flags(skb));
+}
+
+static int key_size(void *sk)
+{
+ return 16; /* Key on DDR */
+}
+
+#define ceil(x, y) \
+ ({ unsigned long __x = (x), __y = (y); (__x + __y - 1) / __y; })
+
+static int data_sgl_len(const struct sk_buff *skb)
+{
+ unsigned int cnt;
+
+ cnt = skb_shinfo(skb)->nr_frags;
+ return (sgl_len(cnt) * 8);
+}
+
+static int nos_ivs(struct sock *sk, unsigned int size)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+
+ return ceil(size, csk->tlshws.mfs);
+}
+
+#define TLS_WR_CPL_LEN \
+ (sizeof(struct fw_tlstx_data_wr) + \
+ sizeof(struct cpl_tx_tls_sfo))
+
+static int is_ivs_imm(struct sock *sk, const struct sk_buff *skb)
+{
+ int ivs_size = nos_ivs(sk, skb->len) * CIPHER_BLOCK_SIZE;
+ int hlen = TLS_WR_CPL_LEN + data_sgl_len(skb);
+
+ if ((hlen + key_size(sk) + ivs_size) <
+ MAX_IMM_OFLD_TX_DATA_WR_LEN) {
+ ULP_SKB_CB(skb)->ulp.tls.iv = 1;
+ return 1;
+ }
+ ULP_SKB_CB(skb)->ulp.tls.iv = 0;
+ return 0;
+}
+
+static int max_ivs_size(struct sock *sk, int size)
+{
+ return (nos_ivs(sk, size) * CIPHER_BLOCK_SIZE);
+}
+
+static int ivs_size(struct sock *sk, const struct sk_buff *skb)
+{
+ return (is_ivs_imm(sk, skb) ? (nos_ivs(sk, skb->len) *
+CIPHER_BLOCK_SIZE) : 0);
+}
+
+static int flowc_wr_credits(int nparams, int *flowclenp)
+{
+ int flowclen16, flowclen;
+
+ flowclen = offsetof(struct fw_flowc_wr, mnemval[nparams]);
+ flowclen16 = DIV_ROUND_UP(flowclen, 16);
+ flowclen = flowclen16 * 16;
+
+ if (flowclenp)
+ *flowclenp = flowclen;
+
+ return flowclen16;
+}
+
+struct sk_buff *create_flowc_wr_skb(struct sock *sk,
+ struct fw_flowc_wr *flowc,
+ int flowclen)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+ struct sk_buff *skb;
+
+ skb = alloc_skb(flowclen, GFP_ATOMIC);
+ if (!skb)
+ return NULL;
+
+ memcpy(__skb_put(skb, flowclen), flowc, flowclen);
+ set_queue(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA, sk);
+
+ return skb;
+}
+
+static int send_flowc_wr(struct sock *sk, struct fw_flowc_wr *flowc,
+int flowclen)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+ bool syn_sent = (sk->sk_state == TCP_SYN_SENT);
+ struct tcp_sock *tp = tcp_sk(sk);
+ int flowclen16 = flowclen / 16;
+ struct sk_buff *skb;
+
+ if (csk_flag(sk, CSK_TX_DATA_SENT)) {
+ skb = create_flowc_wr_skb(sk, flowc, flowclen);
+ if (!skb)
+ return -ENOMEM;
+
+ if (syn_sent)
+ __skb_queue_tail(&csk->ooo_queue, skb);
+ else
+ skb_entail(sk, skb,
+ ULPCB_FLAG_NO_HDR | ULPCB_FLAG_NO_APPEND);
+ return 0;
+ }
+
+ if (!syn_sent) {
+ int ret;
+
+ ret = cxgb4_immdata_send(csk->egress_dev,
+csk->txq_idx,
+flowc, flowclen);
+ if (!ret)
+ return flowclen16;
+ }
+ skb = create_flowc_wr_skb(sk, flowc, flowclen);
+ if (!skb)
+ return -ENOMEM;
+ send_or_def
[RFC crypto v2 5/9] chtls: Key program
Program the tx and rx key on chip.
Signed-off-by: Atul Gupta
---
V2: using aes instead of aes-generic
memset key after programming
---
drivers/crypto/chelsio/chtls/chtls_hw.c | 394
1 file changed, 394 insertions(+)
create mode 100644 drivers/crypto/chelsio/chtls/chtls_hw.c
diff --git a/drivers/crypto/chelsio/chtls/chtls_hw.c
b/drivers/crypto/chelsio/chtls/chtls_hw.c
new file mode 100644
index 000..8b6284a
--- /dev/null
+++ b/drivers/crypto/chelsio/chtls/chtls_hw.c
@@ -0,0 +1,394 @@
+/*
+ * Copyright (c) 2017 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Written by: Atul Gupta (atul.gu...@chelsio.com)
+ */
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "chtls.h"
+#include "chtls_cm.h"
+
+static void __set_tcb_field_direct(struct chtls_sock *csk,
+ struct cpl_set_tcb_field *req, u16 word,
+ u64 mask, u64 val, u8 cookie, int no_reply)
+{
+ struct ulptx_idata *sc;
+
+ INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, csk->tid);
+ req->wr.wr_mid |= htonl(FW_WR_FLOWID_V(csk->tid));
+ req->reply_ctrl = htons(NO_REPLY_V(no_reply) |
+ QUEUENO_V(csk->rss_qid));
+ req->word_cookie = htons(TCB_WORD_V(word) | TCB_COOKIE_V(cookie));
+ req->mask = cpu_to_be64(mask);
+ req->val = cpu_to_be64(val);
+ sc = (struct ulptx_idata *)(req + 1);
+ sc->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP));
+ sc->len = htonl(0);
+}
+
+void __set_tcb_field(struct sock *sk, struct sk_buff *skb, u16 word,
+u64 mask, u64 val, u8 cookie, int no_reply)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+ struct cpl_set_tcb_field *req;
+ struct ulptx_idata *sc;
+ unsigned int wrlen = roundup(sizeof(*req) + sizeof(*sc), 16);
+
+ req = (struct cpl_set_tcb_field *)__skb_put(skb, wrlen);
+ __set_tcb_field_direct(csk, req, word, mask, val, cookie, no_reply);
+ set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id);
+}
+
+static int chtls_set_tcb_field(struct sock *sk, u16 word, u64 mask, u64 val)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+ struct sk_buff *skb;
+ struct cpl_set_tcb_field *req;
+ struct ulptx_idata *sc;
+ unsigned int wrlen = roundup(sizeof(*req) + sizeof(*sc), 16);
+ unsigned int credits_needed = DIV_ROUND_UP(wrlen, 16);
+
+ skb = alloc_skb(wrlen, GFP_ATOMIC);
+ if (!skb)
+ return -ENOMEM;
+
+ __set_tcb_field(sk, skb, word, mask, val, 0, 1);
+ set_queue(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA, sk);
+ csk->wr_credits -= credits_needed;
+ csk->wr_unacked += credits_needed;
+ enqueue_wr(csk, skb);
+ cxgb4_ofld_send(csk->egress_dev, skb);
+ return 0;
+}
+
+/*
+ * Set one of the t_flags bits in the TCB.
+ */
+int chtls_set_tcb_tflag(struct sock *sk, unsigned int bit_pos, int val)
+{
+ return chtls_set_tcb_field(sk, 1, 1ULL << bit_pos,
+ val << bit_pos);
+}
+
+static int chtls_set_tcb_keyid(struct sock *sk, int keyid)
+{
+ return chtls_set_tcb_field(sk, 31, 0xULL, keyid);
+}
+
+static int chtls_set_tcb_seqno(struct sock *sk)
+{
+ return chtls_set_tcb_field(sk, 28, ~0ULL, 0);
+}
+
+static int chtls_set_tcb_quiesce(struct sock *sk, int val)
+{
+ return chtls_set_tcb_field(sk, 1, (1ULL << TF_RX_QUIESCE_S),
+ TF_RX_QUIESCE_V(val));
+}
+
+static void *chtls_alloc_mem(unsigned long size)
+{
+ void *p = kmalloc(size, GFP_KERNEL);
+
+ if (!p)
+ p = vmalloc(size);
+ if (p)
+ memset(p, 0, size);
+ return p;
+}
+
+static void chtls_free_mem(void *addr)
+{
+ unsigned long p = (unsigned long)addr;
+
+ if (p >= VMALLOC_START && p < VMALLOC_END)
+ vfree(addr);
+ else
+ kfree(addr);
+}
+
+/* TLS Key bitmap processing */
+int chtls_init_kmap(struct chtls_dev *cdev, struct cxgb4_lld_info *lldi)
+{
+ unsigned int num_key_ctx, bsize;
+
+ num_key_ctx = (lldi->vr->key.size / TLS_KEY_CONTEXT_SZ);
+ bsize = BITS_TO_LONGS(num_key_ctx);
+
+ cdev->kmap.size = num_key_ctx;
+ cdev->kmap.available = bsize;
+ cdev->kmap.addr = chtls_alloc_mem(sizeof(*cdev->kmap.addr) *
+ bsize);
+ if (!cdev->kmap.addr)
+ return -1;
+
+ cdev->kmap.start = lldi->vr->key.start;
+ spin_lock_init(&cdev->kmap.loc
[RFC crypto v2 6/9] chtls: CPL handler definition
CPL handlers for TLS session, record transmit and receive.
Signed-off-by: Atul Gupta
---
V2: Excess Whitespace
Reverse christmas tree format
skb free in TCP_SYN_RECV
removed !ret check
export tcp_time_wait
fix newsk leak in error case
---
drivers/crypto/chelsio/chtls/chtls_cm.c | 2045 +++
net/ipv4/tcp_minisocks.c|1 +
2 files changed, 2046 insertions(+)
create mode 100644 drivers/crypto/chelsio/chtls/chtls_cm.c
diff --git a/drivers/crypto/chelsio/chtls/chtls_cm.c
b/drivers/crypto/chelsio/chtls/chtls_cm.c
new file mode 100644
index 000..aa98d4e
--- /dev/null
+++ b/drivers/crypto/chelsio/chtls/chtls_cm.c
@@ -0,0 +1,2045 @@
+/*
+ * Copyright (c) 2017 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Written by: Atul Gupta (atul.gu...@chelsio.com)
+ */
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "chtls.h"
+#include "chtls_cm.h"
+
+extern struct request_sock_ops chtls_rsk_ops;
+
+/*
+ * State transitions and actions for close. Note that if we are in SYN_SENT
+ * we remain in that state as we cannot control a connection while it's in
+ * SYN_SENT; such connections are allowed to establish and are then aborted.
+ */
+static unsigned char new_state[16] = {
+ /* current state: new state: action: */
+ /* (Invalid) */ TCP_CLOSE,
+ /* TCP_ESTABLISHED */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
+ /* TCP_SYN_SENT*/ TCP_SYN_SENT,
+ /* TCP_SYN_RECV*/ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
+ /* TCP_FIN_WAIT1 */ TCP_FIN_WAIT1,
+ /* TCP_FIN_WAIT2 */ TCP_FIN_WAIT2,
+ /* TCP_TIME_WAIT */ TCP_CLOSE,
+ /* TCP_CLOSE */ TCP_CLOSE,
+ /* TCP_CLOSE_WAIT */ TCP_LAST_ACK | TCP_ACTION_FIN,
+ /* TCP_LAST_ACK*/ TCP_LAST_ACK,
+ /* TCP_LISTEN */ TCP_CLOSE,
+ /* TCP_CLOSING */ TCP_CLOSING,
+};
+
+static struct chtls_sock *chtls_sock_create(struct chtls_dev *cdev)
+{
+ struct chtls_sock *csk = kzalloc(sizeof(*csk), GFP_ATOMIC);
+
+ if (!csk)
+ return NULL;
+
+ csk->txdata_skb_cache = alloc_skb(TXDATA_SKB_LEN, GFP_ATOMIC);
+ if (!csk->txdata_skb_cache) {
+ kfree(csk);
+ return NULL;
+ }
+
+ kref_init(&csk->kref);
+ csk->cdev = cdev;
+ skb_queue_head_init(&csk->txq);
+ csk->wr_skb_head = NULL;
+ csk->wr_skb_tail = NULL;
+ csk->mss = MAX_MSS;
+ csk->tlshws.ofld = 1;
+ csk->tlshws.txkey = -1;
+ csk->tlshws.rxkey = -1;
+ csk->tlshws.mfs = TLS_MFS;
+ skb_queue_head_init(&csk->tlshws.sk_recv_queue);
+ return csk;
+}
+
+void chtls_sock_release(struct kref *ref)
+{
+ struct chtls_sock *csk =
+ container_of(ref, struct chtls_sock, kref);
+
+ kfree(csk);
+}
+
+static struct net_device *chtls_ipv4_netdev(struct chtls_dev *cdev,
+ struct sock *sk)
+{
+ struct net_device *ndev = cdev->ports[0];
+
+ if (likely(!inet_sk(sk)->inet_rcv_saddr))
+ return ndev;
+
+ ndev = ip_dev_find(&init_net, inet_sk(sk)->inet_rcv_saddr);
+ if (!ndev)
+ return NULL;
+
+ if (is_vlan_dev(ndev))
+ return vlan_dev_real_dev(ndev);
+ return ndev;
+}
+
+static void assign_rxopt(struct sock *sk, unsigned int opt)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+ struct tcp_sock *tp = tcp_sk(sk);
+ const struct chtls_dev *cdev;
+
+ cdev = csk->cdev;
+ tp->tcp_header_len = sizeof(struct tcphdr);
+ tp->rx_opt.mss_clamp = cdev->mtus[TCPOPT_MSS_G(opt)] - 40;
+ tp->mss_cache= tp->rx_opt.mss_clamp;
+ tp->rx_opt.tstamp_ok = TCPOPT_TSTAMP_G(opt);
+ tp->rx_opt.snd_wscale= TCPOPT_SACK_G(opt);
+ tp->rx_opt.wscale_ok = TCPOPT_WSCALE_OK_G(opt);
+ SND_WSCALE(tp) = TCPOPT_SND_WSCALE_G(opt);
+ if (!tp->rx_opt.wscale_ok)
+ tp->rx_opt.rcv_wscale = 0;
+ if (tp->rx_opt.tstamp_ok) {
+ tp->tcp_header_len += TCPOLEN_TSTAMP_ALIGNED;
+ tp->rx_opt.mss_clamp -= TCPOLEN_TSTAMP_ALIGNED;
+ } else if (csk->opt2 & TSTAMPS_EN_F) {
+ csk->opt2 &= ~TSTAMPS_EN_F;
+ csk->mtu_idx = TCPOPT_MSS_G(opt);
+ }
+}
+
+static void chtls_purge_rcv_queue(struct sock *sk)
+{
+ struct sk_buff *skb;
+
+ while ((skb = __skb_dequeue(&sk->sk_receive_queue)) !=
[RFC crypto v2 8/9] chtls: Register the ULP
Add new uld driver for Inline TLS support. Register ULP for chtls.
Setsockopt to program key on chip. support AES GCM key size 128.
Signed-off-by: Atul Gupta
---
drivers/crypto/chelsio/chtls/chtls_main.c | 584 ++
include/uapi/linux/tls.h | 1 +
2 files changed, 585 insertions(+)
create mode 100644 drivers/crypto/chelsio/chtls/chtls_main.c
diff --git a/drivers/crypto/chelsio/chtls/chtls_main.c
b/drivers/crypto/chelsio/chtls/chtls_main.c
new file mode 100644
index 000..7ccbc8d
--- /dev/null
+++ b/drivers/crypto/chelsio/chtls/chtls_main.c
@@ -0,0 +1,584 @@
+/*
+ * Copyright (c) 2017 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Written by: Atul Gupta (atul.gu...@chelsio.com)
+ */
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "chtls.h"
+#include "chtls_cm.h"
+
+#define DRV_NAME "chtls"
+
+/*
+ * chtls device management
+ * maintains a list of the chtls devices
+ */
+static LIST_HEAD(cdev_list);
+static DEFINE_MUTEX(cdev_mutex);
+static DEFINE_MUTEX(cdev_list_lock);
+
+static struct proto chtls_base_prot;
+static struct proto chtls_cpl_prot;
+static DEFINE_MUTEX(notify_mutex);
+static RAW_NOTIFIER_HEAD(listen_notify_list);
+struct request_sock_ops chtls_rsk_ops;
+static uint send_page_order = (14 - PAGE_SHIFT < 0) ? 0 : 14 - PAGE_SHIFT;
+
+int register_listen_notifier(struct notifier_block *nb)
+{
+ int err;
+
+ mutex_lock(¬ify_mutex);
+ err = raw_notifier_chain_register(&listen_notify_list, nb);
+ mutex_unlock(¬ify_mutex);
+ return err;
+}
+
+int unregister_listen_notifier(struct notifier_block *nb)
+{
+ int err;
+
+ mutex_lock(¬ify_mutex);
+ err = raw_notifier_chain_unregister(&listen_notify_list, nb);
+ mutex_unlock(¬ify_mutex);
+ return err;
+}
+
+int listen_notify_handler(struct notifier_block *this,
+ unsigned long event, void *data)
+{
+ struct sock *sk = data;
+ struct chtls_dev *cdev;
+ int ret = NOTIFY_DONE;
+
+ switch (event) {
+ case CHTLS_LISTEN_START:
+ case CHTLS_LISTEN_STOP:
+ mutex_lock(&cdev_list_lock);
+ list_for_each_entry(cdev, &cdev_list, list) {
+ if (event == CHTLS_LISTEN_START)
+ ret = chtls_listen_start(cdev, sk);
+ else
+ chtls_listen_stop(cdev, sk);
+ }
+ mutex_unlock(&cdev_list_lock);
+ break;
+ }
+ return ret;
+}
+
+static struct notifier_block listen_notifier = {
+ .notifier_call = listen_notify_handler
+};
+
+static int listen_backlog_rcv(struct sock *sk, struct sk_buff *skb)
+{
+ if (likely(skb_transport_header(skb) != skb_network_header(skb)))
+ return tcp_v4_do_rcv(sk, skb);
+ BLOG_SKB_CB(skb)->backlog_rcv(sk, skb);
+ return 0;
+}
+
+static int chtls_start_listen(struct sock *sk)
+{
+ int err;
+
+ if (sk->sk_protocol != IPPROTO_TCP)
+ return -EPROTONOSUPPORT;
+
+ if (sk->sk_family == PF_INET &&
+ LOOPBACK(inet_sk(sk)->inet_rcv_saddr))
+ return -EADDRNOTAVAIL;
+
+ sk->sk_backlog_rcv = listen_backlog_rcv;
+ mutex_lock(¬ify_mutex);
+ err = raw_notifier_call_chain(&listen_notify_list, 0, sk);
+ mutex_unlock(¬ify_mutex);
+ return err;
+}
+
+static int chtls_hash(struct sock *sk)
+{
+ int err;
+
+ err = tcp_prot.hash(sk);
+ if (sk->sk_state == TCP_LISTEN)
+ err |= chtls_start_listen(sk);
+
+ if (err)
+ tcp_prot.unhash(sk);
+ return err;
+}
+
+int chtls_stop_listen(struct sock *sk)
+{
+ if (sk->sk_protocol != IPPROTO_TCP)
+ return -EPROTONOSUPPORT;
+
+ mutex_lock(¬ify_mutex);
+ raw_notifier_call_chain(&listen_notify_list, 1, sk);
+ mutex_unlock(¬ify_mutex);
+ return 0;
+}
+
+static void chtls_unhash(struct sock *sk)
+{
+ if (sk->sk_state == TCP_LISTEN)
+ chtls_stop_listen(sk);
+ tcp_prot.unhash(sk);
+}
+
+static void chtls_lsk_close(struct sock *sk, long timeout)
+{
+ struct tls_context *ctx = tls_get_ctx(sk);
+ void (*sk_proto_close)(struct sock *sk, long timeout);
+
+ lock_sock(sk);
+ sk_proto_close = ctx->sk_proto_close;
+ kfree(ctx);
+
+ release_sock(sk);
+ sk_proto_close(sk, timeout);
+}
+
+static void process_deferq(struct work_struct *task_param)
+{
+ struct chtls_dev *cdev = container_of(task_param,
+ struct chtls_dev, deferq_tas
[RFC crypto v2 9/9] Makefile Kconfig
Entry for Inline TLS as another driver dependent on cxgb4 and chcr Signed-off-by: Atul Gupta --- drivers/crypto/chelsio/Kconfig| 10 ++ drivers/crypto/chelsio/Makefile | 1 + drivers/crypto/chelsio/chtls/Makefile | 4 3 files changed, 15 insertions(+) create mode 100644 drivers/crypto/chelsio/chtls/Makefile diff --git a/drivers/crypto/chelsio/Kconfig b/drivers/crypto/chelsio/Kconfig index 51932c7..686d246 100644 --- a/drivers/crypto/chelsio/Kconfig +++ b/drivers/crypto/chelsio/Kconfig @@ -28,3 +28,13 @@ config CHELSIO_IPSEC_INLINE default n ---help--- Enable support for IPSec Tx Inline. + +config CRYPTO_DEV_CHELSIO_TLS +tristate "Chelsio Crypto Inline TLS Driver" +depends on CHELSIO_T4 +select CRYPTO_DEV_CHELSIO +---help--- + Support Chelsio Inline TLS with Chelsio crypto accelerator. + + To compile this driver as a module, choose M here: the module + will be called chtls. diff --git a/drivers/crypto/chelsio/Makefile b/drivers/crypto/chelsio/Makefile index eaecaf1..639e571 100644 --- a/drivers/crypto/chelsio/Makefile +++ b/drivers/crypto/chelsio/Makefile @@ -3,3 +3,4 @@ ccflags-y := -Idrivers/net/ethernet/chelsio/cxgb4 obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chcr.o chcr-objs := chcr_core.o chcr_algo.o chcr-$(CONFIG_CHELSIO_IPSEC_INLINE) += chcr_ipsec.o +obj-$(CONFIG_CRYPTO_DEV_CHELSIO_TLS) += chtls/ diff --git a/drivers/crypto/chelsio/chtls/Makefile b/drivers/crypto/chelsio/chtls/Makefile new file mode 100644 index 000..df13795 --- /dev/null +++ b/drivers/crypto/chelsio/chtls/Makefile @@ -0,0 +1,4 @@ +ccflags-y := -Idrivers/net/ethernet/chelsio/cxgb4 -Idrivers/crypto/chelsio/ + +obj-$(CONFIG_CRYPTO_DEV_CHELSIO_TLS) += chtls.o +chtls-objs := chtls_main.o chtls_cm.o chtls_io.o chtls_hw.o -- 1.8.3.1
[RFC crypto v3 1/9] chtls: structure and macro definiton
Inline TLS state, connection management. Supporting macros definition.
Signed-off-by: Atul Gupta
---
drivers/crypto/chelsio/chtls/chtls.h| 480
drivers/crypto/chelsio/chtls/chtls_cm.h | 203 ++
2 files changed, 683 insertions(+)
create mode 100644 drivers/crypto/chelsio/chtls/chtls.h
create mode 100644 drivers/crypto/chelsio/chtls/chtls_cm.h
diff --git a/drivers/crypto/chelsio/chtls/chtls.h
b/drivers/crypto/chelsio/chtls/chtls.h
new file mode 100644
index 000..33d3ed4
--- /dev/null
+++ b/drivers/crypto/chelsio/chtls/chtls.h
@@ -0,0 +1,480 @@
+/*
+ * Copyright (c) 2016 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef __CHTLS_H__
+#define __CHTLS_H__
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "t4fw_api.h"
+#include "t4_msg.h"
+#include "cxgb4.h"
+#include "cxgb4_uld.h"
+#include "l2t.h"
+#include "chcr_algo.h"
+#include "chcr_core.h"
+#include "chcr_crypto.h"
+
+#define CIPHER_BLOCK_SIZE 16
+#define MAX_IVS_PAGE256
+#define TLS_KEY_CONTEXT_SZ 64
+#define TLS_HEADER_LENGTH 5
+#define SCMD_CIPH_MODE_AES_GCM 2
+#define GCM_TAG_SIZE16
+#define AEAD_EXPLICIT_DATA_SIZE 8
+/* Any MFS size should work and come from openssl */
+#define TLS_MFS16384
+
+#define SOCK_INLINE (31)
+#define RSS_HDR sizeof(struct rss_header)
+
+enum {
+ CHTLS_KEY_CONTEXT_DSGL,
+ CHTLS_KEY_CONTEXT_IMM,
+ CHTLS_KEY_CONTEXT_DDR,
+};
+
+enum {
+ CHTLS_LISTEN_START,
+ CHTLS_LISTEN_STOP,
+};
+
+/* Flags for return value of CPL message handlers */
+enum {
+ CPL_RET_BUF_DONE = 1, /* buffer processing done */
+ CPL_RET_BAD_MSG = 2,/* bad CPL message */
+ CPL_RET_UNKNOWN_TID = 4 /* unexpected unknown TID */
+};
+
+#define TLS_RCV_ST_READ_HEADER 0xF0
+#define TLS_RCV_ST_READ_BODY0xF1
+#define TLS_RCV_ST_READ_DONE0xF2
+#define TLS_RCV_ST_READ_NB 0xF3
+
+#define RSPQ_HASH_BITS 5
+#define LISTEN_INFO_HASH_SIZE 32
+struct listen_info {
+ struct listen_info *next; /* Link to next entry */
+ struct sock *sk; /* The listening socket */
+ unsigned int stid; /* The server TID */
+};
+
+enum {
+ T4_LISTEN_START_PENDING,
+ T4_LISTEN_STARTED
+};
+
+enum csk_flags {
+ CSK_CALLBACKS_CHKD, /* socket callbacks have been sanitized */
+ CSK_ABORT_REQ_RCVD, /* received one ABORT_REQ_RSS message */
+ CSK_TX_MORE_DATA, /* sending ULP data; don't set SHOVE bit */
+ CSK_TX_WAIT_IDLE, /* suspend Tx until in-flight data is ACKed */
+ CSK_ABORT_SHUTDOWN, /* shouldn't send more abort requests */
+ CSK_ABORT_RPL_PENDING, /* expecting an abort reply */
+ CSK_CLOSE_CON_REQUESTED,/* we've sent a close_conn_req */
+ CSK_TX_DATA_SENT, /* sent a TX_DATA WR on this connection */
+ CSK_TX_FAILOVER,/* Tx traffic failing over */
+ CSK_UPDATE_RCV_WND, /* Need to update rcv window */
+ CSK_RST_ABORTED,/* outgoing RST was aborted */
+ CSK_TLS_HANDSHK,/* TLS Handshake */
+};
+
+struct listen_ctx {
+ struct sock *lsk;
+ struct chtls_dev *cdev;
+ u32 state;
+};
+
+struct key_map {
+ unsigned long *addr;
+ unsigned int start;
+ unsigned int available;
+ unsigned int size;
+ spinlock_t lock; /* lock for key id request from map */
+} __packed;
+
+struct tls_scmd {
+ __be32 seqno_numivs;
+ __be32 ivgen_hdrlen;
+};
+
+struct chtls_dev {
+ struct list_head list;
+ struct cxgb4_lld_info *lldi;
+ struct pci_dev *pdev;
+ struct listen_info *listen_hash_tab[LISTEN_INFO_HASH_SIZE];
+ spinlock_t listen_lock; /* lock for listen list */
+ struct net_device **ports;
+ struct tid_info *tids;
+ unsigned int pfvf;
+ const unsigned short *mtus;
+
+ spinlock_t aidr_lock cacheline_aligned_in_smp;
+ struct idr aidr; /* ATID id space */
+ struct idr hwtid_idr;
+ struct idr stid_idr;
+
+ spinlock_t idr_lock cacheline_aligned_in_smp;
+
+ struct net_device *egr_dev[NCHAN * 2];
+ struct sk_buff *rspq_skb_cache[1 << RSPQ_HASH_BITS];
+ struct sk_buff *askb;
+
+ struct sk_buff_head deferq;
+ struct work_struct deferq_task;
+
+ struct list_head list_node;
+ struct list_head rcu_node;
+ struct list_head na_node;
+ unsigned int send_page_order;
+
[RFC crypto v3 0/9] Chelsio Inline TLS
RFC series for Chelsio Inline TLS driver (chtls.ko)
Driver use the ULP infrastructure to register chtls as Inline TLS ULP.
Chtls use TCP Sockets to transmit and receive TLS record. TCP proto_ops
is extended to offload TLS record.
T6 adapter provides the following features:
-TLS record offload, TLS header, encrypt, digest and transmit
-TLS record receive and decrypt
-TLS keys store
-TCP/IP engine
-TLS engine
-GCM crypto engine [support CBC also]
TLS provides security at the transport layer. It uses TCP to provide
reliable end-to-end transport of application data. It relies on TCP
for any retransmission. TLS session comprises of three parts:
a. TCP/IP connection
b. TLS handshake
c. Record layer processing
TLS handshake state machine is executed in host (refer standard
implementation eg. OpenSSL). Setsockopt [SOL_TCP, TCP_ULP] initialize
TCP proto-ops for Chelsio inline tls support. setsockopt(sock, SOL_TCP,
TCP_ULP, "chtls", sizeof("chtls"));
Tx and Rx Keys are decided during handshake and programmed onto the chip
after CCS is exchanged.
struct tls12_crypto_info_aes_gcm_128 crypto_info
setsockopt(sock, SOL_TLS, TLS_TX, &crypto_info, sizeof(crypto_info))
Finish is the first encrypted/decrypted message tx/rx inline.
On the Tx path TLS engine receive plain text from openssl, insert IV,
fetches the tx key, create cipher text records and generate MAC. TLS
header is added to cipher text and forward to TCP/IP engine for transport
layer processing and transmission on wire.
TX:
Application--openssl--chtls---TLS engine---encrypt/auth---TCP/IP
engine---wire.
On the Rx side, data received is PDU aligned at record
boundaries. TLS processes only the complete record. If rx key is programmed
on CCS receive, data is decrypted and plain text is posted to host.
RX:
Wire--cipher-text--TCP/IP engine [PDU align]---TLS engine---
decrypt/auth---plain-text--chtls--openssl--application
v3: fixed the kbuild test issues
-made few funtions static
-initialized few variables
v2: fixed the following based on the review comments of Stephan Mueller,
Stefano Brivio and Hannes Frederic
-Added more details in cover letter
-Fixed indentation and formating issues
-Using aes instead of aes-generic
-memset key info after programing the key on chip
-reordered the patch sequence
Atul Gupta (9):
chtls: structure and macro definiton
cxgb4: Inline TLS FW Interface
cxgb4: LLD driver changes to enable TLS
chcr: Key Macro
chtls: Key program
chtls: CPL handler definition
chtls: Inline crypto request Tx/Rx
chtls: Register the ULP
Makefile Kconfig
drivers/crypto/chelsio/Kconfig | 10 +
drivers/crypto/chelsio/Makefile|1 +
drivers/crypto/chelsio/chcr_algo.h | 42 +
drivers/crypto/chelsio/chcr_core.h | 55 +-
drivers/crypto/chelsio/chtls/Makefile |4 +
drivers/crypto/chelsio/chtls/chtls.h | 480 +
drivers/crypto/chelsio/chtls/chtls_cm.c| 2045
drivers/crypto/chelsio/chtls/chtls_cm.h| 203 ++
drivers/crypto/chelsio/chtls/chtls_hw.c| 394
drivers/crypto/chelsio/chtls/chtls_io.c| 1867 ++
drivers/crypto/chelsio/chtls/chtls_main.c | 584 ++
drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c | 18 +-
drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c| 32 +-
drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h |7 +
drivers/net/ethernet/chelsio/cxgb4/sge.c | 98 +-
drivers/net/ethernet/chelsio/cxgb4/t4_msg.h| 121 +-
drivers/net/ethernet/chelsio/cxgb4/t4_regs.h |2 +
drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h | 165 +-
include/uapi/linux/tls.h |1 +
net/ipv4/tcp_minisocks.c |1 +
20 files changed, 6111 insertions(+), 19 deletions(-)
create mode 100644 drivers/crypto/chelsio/chtls/Makefile
create mode 100644 drivers/crypto/chelsio/chtls/chtls.h
create mode 100644 drivers/crypto/chelsio/chtls/chtls_cm.c
create mode 100644 drivers/crypto/chelsio/chtls/chtls_cm.h
create mode 100644 drivers/crypto/chelsio/chtls/chtls_hw.c
create mode 100644 drivers/crypto/chelsio/chtls/chtls_io.c
create mode 100644 drivers/crypto/chelsio/chtls/chtls_main.c
--
1.8.3.1
[RFC crypto v3 2/9] cxgb4: Inline TLS FW Interface
Key area size in hw-config file. CPL struct for TLS request
and response. Work request for Inline TLS.
Signed-off-by: Atul Gupta
---
drivers/net/ethernet/chelsio/cxgb4/t4_msg.h | 121 ++-
drivers/net/ethernet/chelsio/cxgb4/t4_regs.h | 2 +
drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h | 165 +-
3 files changed, 283 insertions(+), 5 deletions(-)
diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h
b/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h
index 7e12f24..9a56e0d 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h
@@ -81,6 +81,7 @@ enum {
CPL_RX_ISCSI_CMP = 0x45,
CPL_TRACE_PKT_T5 = 0x48,
CPL_RX_ISCSI_DDP = 0x49,
+ CPL_RX_TLS_CMP= 0x4E,
CPL_RDMA_READ_REQ = 0x60,
@@ -88,6 +89,7 @@ enum {
CPL_ACT_OPEN_REQ6 = 0x83,
CPL_TX_TLS_PDU =0x88,
+ CPL_TX_TLS_SFO= 0x89,
CPL_TX_SEC_PDU= 0x8A,
CPL_TX_TLS_ACK= 0x8B,
@@ -97,6 +99,7 @@ enum {
CPL_RX_MPS_PKT= 0xAF,
CPL_TRACE_PKT = 0xB0,
+ CPL_TLS_DATA = 0xB1,
CPL_ISCSI_DATA= 0xB2,
CPL_FW4_MSG = 0xC0,
@@ -150,6 +153,7 @@ enum {
ULP_MODE_RDMA = 4,
ULP_MODE_TCPDDP= 5,
ULP_MODE_FCOE = 6,
+ ULP_MODE_TLS = 8,
};
enum {
@@ -1414,6 +1418,14 @@ struct cpl_tx_data {
#define TX_FORCE_S 13
#define TX_FORCE_V(x) ((x) << TX_FORCE_S)
+#define TX_SHOVE_S14
+#define TX_SHOVE_V(x) ((x) << TX_SHOVE_S)
+
+#define TX_ULP_MODE_S10
+#define TX_ULP_MODE_M0x7
+#define TX_ULP_MODE_V(x) ((x) << TX_ULP_MODE_S)
+#define TX_ULP_MODE_G(x) (((x) >> TX_ULP_MODE_S) & TX_ULP_MODE_M)
+
#define T6_TX_FORCE_S 20
#define T6_TX_FORCE_V(x) ((x) << T6_TX_FORCE_S)
#define T6_TX_FORCE_F T6_TX_FORCE_V(1U)
@@ -1428,12 +1440,21 @@ enum {
ULP_TX_SC_NOOP = 0x80,
ULP_TX_SC_IMM = 0x81,
ULP_TX_SC_DSGL = 0x82,
- ULP_TX_SC_ISGL = 0x83
+ ULP_TX_SC_ISGL = 0x83,
+ ULP_TX_SC_MEMRD = 0x86
};
#define ULPTX_CMD_S24
#define ULPTX_CMD_V(x) ((x) << ULPTX_CMD_S)
+#define ULPTX_LEN16_S0
+#define ULPTX_LEN16_M0xFF
+#define ULPTX_LEN16_V(x) ((x) << ULPTX_LEN16_S)
+
+#define ULP_TX_SC_MORE_S 23
+#define ULP_TX_SC_MORE_V(x) ((x) << ULP_TX_SC_MORE_S)
+#define ULP_TX_SC_MORE_F ULP_TX_SC_MORE_V(1U)
+
struct ulptx_sge_pair {
__be32 len[2];
__be64 addr[2];
@@ -1948,4 +1969,102 @@ enum {
X_CPL_RX_MPS_PKT_TYPE_QFC = 1 << 2,
X_CPL_RX_MPS_PKT_TYPE_PTP = 1 << 3
};
+
+struct cpl_tx_tls_sfo {
+ __be32 op_to_seg_len;
+ __be32 pld_len;
+ __be32 type_protover;
+ __be32 r1_lo;
+ __be32 seqno_numivs;
+ __be32 ivgen_hdrlen;
+ __be64 scmd1;
+};
+
+/* cpl_tx_tls_sfo macros */
+#define CPL_TX_TLS_SFO_OPCODE_S 24
+#define CPL_TX_TLS_SFO_OPCODE_V(x) ((x) << CPL_TX_TLS_SFO_OPCODE_S)
+
+#define CPL_TX_TLS_SFO_DATA_TYPE_S 20
+#define CPL_TX_TLS_SFO_DATA_TYPE_V(x) ((x) << CPL_TX_TLS_SFO_DATA_TYPE_S)
+
+#define CPL_TX_TLS_SFO_CPL_LEN_S16
+#define CPL_TX_TLS_SFO_CPL_LEN_V(x) ((x) << CPL_TX_TLS_SFO_CPL_LEN_S)
+
+#define CPL_TX_TLS_SFO_SEG_LEN_S0
+#define CPL_TX_TLS_SFO_SEG_LEN_M0x
+#define CPL_TX_TLS_SFO_SEG_LEN_V(x) ((x) << CPL_TX_TLS_SFO_SEG_LEN_S)
+#define CPL_TX_TLS_SFO_SEG_LEN_G(x) \
+ (((x) >> CPL_TX_TLS_SFO_SEG_LEN_S) & CPL_TX_TLS_SFO_SEG_LEN_M)
+
+#define CPL_TX_TLS_SFO_TYPE_S 24
+#define CPL_TX_TLS_SFO_TYPE_M 0xff
+#define CPL_TX_TLS_SFO_TYPE_V(x)((x) << CPL_TX_TLS_SFO_TYPE_S)
+#define CPL_TX_TLS_SFO_TYPE_G(x)\
+ (((x) >> CPL_TX_TLS_SFO_TYPE_S) & CPL_TX_TLS_SFO_TYPE_M)
+
+#define CPL_TX_TLS_SFO_PROTOVER_S 8
+#define CPL_TX_TLS_SFO_PROTOVER_M 0x
+#define CPL_TX_TLS_SFO_PROTOVER_V(x)((x) << CPL_TX_TLS_SFO_PROTOVER_S)
+#define CPL_TX_TLS_SFO_PROTOVER_G(x)\
+ (((x) >> CPL_TX_TLS_SFO_PROTOVER_S) & CPL_TX_TLS_SFO_PROTOVER_M)
+
+struct cpl_tls_data {
+ struct rss_header rsshdr;
+ union opcode_tid ot;
+ __be32 length_pkd;
+ __be32 seq;
+ __be32 r1;
+};
+
+#define CPL_TLS_DATA_OPCODE_S 24
+#define CPL_TLS_DATA_OPCODE_M 0xff
+#define CPL_TLS_DATA_OPCODE_V(x)((x) << CPL_TLS_DATA_OPCODE_S)
+#define CPL_TLS_DATA_OPCODE_G(x)\
+ (((x) >> CPL_TLS_DATA_OPCODE_S) & CPL_TLS_DATA_OPCODE_M)
+
+#define CPL_TLS_DATA_TID_S 0
+#define CPL_TLS_DATA_TID_M 0xff
+#define CPL_TLS_DATA_TID_V(x) ((x) << CPL_TLS_DATA_TID_S)
+#define CPL_TLS_DATA_TID_G(x) \
+ (((x) >> CPL_TLS_DATA_TID_S) &
[RFC crypto v3 5/9] chtls: Key program
Program the tx and rx key on chip.
Signed-off-by: Atul Gupta
---
v3: made some functions static
---
drivers/crypto/chelsio/chtls/chtls_hw.c | 394
1 file changed, 394 insertions(+)
create mode 100644 drivers/crypto/chelsio/chtls/chtls_hw.c
diff --git a/drivers/crypto/chelsio/chtls/chtls_hw.c
b/drivers/crypto/chelsio/chtls/chtls_hw.c
new file mode 100644
index 000..c3e17159
--- /dev/null
+++ b/drivers/crypto/chelsio/chtls/chtls_hw.c
@@ -0,0 +1,394 @@
+/*
+ * Copyright (c) 2017 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Written by: Atul Gupta (atul.gu...@chelsio.com)
+ */
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "chtls.h"
+#include "chtls_cm.h"
+
+static void __set_tcb_field_direct(struct chtls_sock *csk,
+ struct cpl_set_tcb_field *req, u16 word,
+ u64 mask, u64 val, u8 cookie, int no_reply)
+{
+ struct ulptx_idata *sc;
+
+ INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, csk->tid);
+ req->wr.wr_mid |= htonl(FW_WR_FLOWID_V(csk->tid));
+ req->reply_ctrl = htons(NO_REPLY_V(no_reply) |
+ QUEUENO_V(csk->rss_qid));
+ req->word_cookie = htons(TCB_WORD_V(word) | TCB_COOKIE_V(cookie));
+ req->mask = cpu_to_be64(mask);
+ req->val = cpu_to_be64(val);
+ sc = (struct ulptx_idata *)(req + 1);
+ sc->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP));
+ sc->len = htonl(0);
+}
+
+static void __set_tcb_field(struct sock *sk, struct sk_buff *skb, u16 word,
+ u64 mask, u64 val, u8 cookie, int no_reply)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+ struct cpl_set_tcb_field *req;
+ struct ulptx_idata *sc;
+ unsigned int wrlen = roundup(sizeof(*req) + sizeof(*sc), 16);
+
+ req = (struct cpl_set_tcb_field *)__skb_put(skb, wrlen);
+ __set_tcb_field_direct(csk, req, word, mask, val, cookie, no_reply);
+ set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id);
+}
+
+static int chtls_set_tcb_field(struct sock *sk, u16 word, u64 mask, u64 val)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+ struct sk_buff *skb;
+ struct cpl_set_tcb_field *req;
+ struct ulptx_idata *sc;
+ unsigned int wrlen = roundup(sizeof(*req) + sizeof(*sc), 16);
+ unsigned int credits_needed = DIV_ROUND_UP(wrlen, 16);
+
+ skb = alloc_skb(wrlen, GFP_ATOMIC);
+ if (!skb)
+ return -ENOMEM;
+
+ __set_tcb_field(sk, skb, word, mask, val, 0, 1);
+ set_queue(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA, sk);
+ csk->wr_credits -= credits_needed;
+ csk->wr_unacked += credits_needed;
+ enqueue_wr(csk, skb);
+ cxgb4_ofld_send(csk->egress_dev, skb);
+ return 0;
+}
+
+/*
+ * Set one of the t_flags bits in the TCB.
+ */
+int chtls_set_tcb_tflag(struct sock *sk, unsigned int bit_pos, int val)
+{
+ return chtls_set_tcb_field(sk, 1, 1ULL << bit_pos,
+ val << bit_pos);
+}
+
+static int chtls_set_tcb_keyid(struct sock *sk, int keyid)
+{
+ return chtls_set_tcb_field(sk, 31, 0xULL, keyid);
+}
+
+static int chtls_set_tcb_seqno(struct sock *sk)
+{
+ return chtls_set_tcb_field(sk, 28, ~0ULL, 0);
+}
+
+static int chtls_set_tcb_quiesce(struct sock *sk, int val)
+{
+ return chtls_set_tcb_field(sk, 1, (1ULL << TF_RX_QUIESCE_S),
+ TF_RX_QUIESCE_V(val));
+}
+
+static void *chtls_alloc_mem(unsigned long size)
+{
+ void *p = kmalloc(size, GFP_KERNEL);
+
+ if (!p)
+ p = vmalloc(size);
+ if (p)
+ memset(p, 0, size);
+ return p;
+}
+
+static void chtls_free_mem(void *addr)
+{
+ unsigned long p = (unsigned long)addr;
+
+ if (p >= VMALLOC_START && p < VMALLOC_END)
+ vfree(addr);
+ else
+ kfree(addr);
+}
+
+/* TLS Key bitmap processing */
+int chtls_init_kmap(struct chtls_dev *cdev, struct cxgb4_lld_info *lldi)
+{
+ unsigned int num_key_ctx, bsize;
+
+ num_key_ctx = (lldi->vr->key.size / TLS_KEY_CONTEXT_SZ);
+ bsize = BITS_TO_LONGS(num_key_ctx);
+
+ cdev->kmap.size = num_key_ctx;
+ cdev->kmap.available = bsize;
+ cdev->kmap.addr = chtls_alloc_mem(sizeof(*cdev->kmap.addr) *
+ bsize);
+ if (!cdev->kmap.addr)
+ return -1;
+
+ cdev->kmap.start = lldi->vr->key.start;
+ spin_lock_init(&cdev->kmap.lock);
+
[RFC crypto v3 3/9] cxgb4: LLD driver changes to enable TLS
Read FW capability. Read key area size. Dump the TLS record count.
Signed-off-by: Atul Gupta
---
drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c | 18 +++-
drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c| 32 +--
drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h | 7 ++
drivers/net/ethernet/chelsio/cxgb4/sge.c | 98 +-
4 files changed, 142 insertions(+), 13 deletions(-)
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
index cf47183..cfc9210 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
@@ -2826,8 +2826,8 @@ static int meminfo_show(struct seq_file *seq, void *v)
"Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
"TDDP region:", "TPT region:", "STAG region:", "RQ region:",
"RQUDP region:", "PBL region:", "TXPBL region:",
- "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
- "On-chip queues:"
+ "TLSKey region:", "DBVFIFO region:", "ULPRX state:",
+ "ULPTX state:", "On-chip queues:"
};
int i, n;
@@ -2943,6 +2943,12 @@ static int meminfo_show(struct seq_file *seq, void *v)
ulp_region(RX_RQUDP);
ulp_region(RX_PBL);
ulp_region(TX_PBL);
+ if (adap->params.crypto & FW_CAPS_CONFIG_TLS_INLINE) {
+ ulp_region(RX_TLS_KEY);
+ } else {
+ md->base = 0;
+ md->idx = ARRAY_SIZE(region);
+ }
#undef ulp_region
md->base = 0;
md->idx = ARRAY_SIZE(region);
@@ -3098,6 +3104,14 @@ static int chcr_show(struct seq_file *seq, void *v)
atomic_read(&adap->chcr_stats.fallback));
seq_printf(seq, "IPSec PDU: %10u\n",
atomic_read(&adap->chcr_stats.ipsec_cnt));
+
+ seq_puts(seq, "\nChelsio Inline TLS Stats\n");
+ seq_printf(seq, "TLS PDU Tx: %u\n",
+ atomic_read(&adap->chcr_stats.tls_pdu_tx));
+ seq_printf(seq, "TLS PDU Rx: %u\n",
+ atomic_read(&adap->chcr_stats.tls_pdu_rx));
+ seq_printf(seq, "TLS Keys (DDR) Count: %u\n",
+ atomic_read(&adap->chcr_stats.tls_key));
return 0;
}
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
index 05a4abf..60eb18b 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
@@ -4086,18 +4086,32 @@ static int adap_init0(struct adapter *adap)
adap->num_ofld_uld += 2;
}
if (caps_cmd.cryptocaps) {
- /* Should query params here...TODO */
- params[0] = FW_PARAM_PFVF(NCRYPTO_LOOKASIDE);
- ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2,
- params, val);
- if (ret < 0) {
- if (ret != -EINVAL)
+ if (ntohs(caps_cmd.cryptocaps) &
+ FW_CAPS_CONFIG_CRYPTO_LOOKASIDE) {
+ params[0] = FW_PARAM_PFVF(NCRYPTO_LOOKASIDE);
+ ret = t4_query_params(adap, adap->mbox, adap->pf, 0,
+ 2, params, val);
+ if (ret < 0) {
+ if (ret != -EINVAL)
+ goto bye;
+ } else {
+ adap->vres.ncrypto_fc = val[0];
+ }
+ adap->num_ofld_uld += 1;
+ }
+ if (ntohs(caps_cmd.cryptocaps) &
+ FW_CAPS_CONFIG_TLS_INLINE) {
+ params[0] = FW_PARAM_PFVF(TLS_START);
+ params[1] = FW_PARAM_PFVF(TLS_END);
+ ret = t4_query_params(adap, adap->mbox, adap->pf, 0,
+ 2, params, val);
+ if (ret < 0)
goto bye;
- } else {
- adap->vres.ncrypto_fc = val[0];
+ adap->vres.key.start = val[0];
+ adap->vres.key.size = val[1] - val[0] + 1;
+ adap->num_uld += 1;
}
adap->params.crypto = ntohs(caps_cmd.cryptocaps);
- adap->num_uld += 1;
}
#undef FW_PARAM_PFVF
#undef FW_PARAM_DEV
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h
b/drivers/
[RFC crypto v3 4/9] chcr: Key Macro
Define macro for TLS Key context
Signed-off-by: Atul Gupta
---
drivers/crypto/chelsio/chcr_algo.h | 42 +
drivers/crypto/chelsio/chcr_core.h | 55 +-
2 files changed, 96 insertions(+), 1 deletion(-)
diff --git a/drivers/crypto/chelsio/chcr_algo.h
b/drivers/crypto/chelsio/chcr_algo.h
index d1673a5..f263cd4 100644
--- a/drivers/crypto/chelsio/chcr_algo.h
+++ b/drivers/crypto/chelsio/chcr_algo.h
@@ -86,6 +86,39 @@
KEY_CONTEXT_OPAD_PRESENT_M)
#define KEY_CONTEXT_OPAD_PRESENT_F KEY_CONTEXT_OPAD_PRESENT_V(1U)
+#define TLS_KEYCTX_RXFLIT_CNT_S 24
+#define TLS_KEYCTX_RXFLIT_CNT_V(x) ((x) << TLS_KEYCTX_RXFLIT_CNT_S)
+
+#define TLS_KEYCTX_RXPROT_VER_S 20
+#define TLS_KEYCTX_RXPROT_VER_M 0xf
+#define TLS_KEYCTX_RXPROT_VER_V(x) ((x) << TLS_KEYCTX_RXPROT_VER_S)
+
+#define TLS_KEYCTX_RXCIPH_MODE_S 16
+#define TLS_KEYCTX_RXCIPH_MODE_M 0xf
+#define TLS_KEYCTX_RXCIPH_MODE_V(x) ((x) << TLS_KEYCTX_RXCIPH_MODE_S)
+
+#define TLS_KEYCTX_RXAUTH_MODE_S 12
+#define TLS_KEYCTX_RXAUTH_MODE_M 0xf
+#define TLS_KEYCTX_RXAUTH_MODE_V(x) ((x) << TLS_KEYCTX_RXAUTH_MODE_S)
+
+#define TLS_KEYCTX_RXCIAU_CTRL_S 11
+#define TLS_KEYCTX_RXCIAU_CTRL_V(x) ((x) << TLS_KEYCTX_RXCIAU_CTRL_S)
+
+#define TLS_KEYCTX_RX_SEQCTR_S 9
+#define TLS_KEYCTX_RX_SEQCTR_M 0x3
+#define TLS_KEYCTX_RX_SEQCTR_V(x) ((x) << TLS_KEYCTX_RX_SEQCTR_S)
+
+#define TLS_KEYCTX_RX_VALID_S 8
+#define TLS_KEYCTX_RX_VALID_V(x) ((x) << TLS_KEYCTX_RX_VALID_S)
+
+#define TLS_KEYCTX_RXCK_SIZE_S 3
+#define TLS_KEYCTX_RXCK_SIZE_M 0x7
+#define TLS_KEYCTX_RXCK_SIZE_V(x) ((x) << TLS_KEYCTX_RXCK_SIZE_S)
+
+#define TLS_KEYCTX_RXMK_SIZE_S 0
+#define TLS_KEYCTX_RXMK_SIZE_M 0x7
+#define TLS_KEYCTX_RXMK_SIZE_V(x) ((x) << TLS_KEYCTX_RXMK_SIZE_S)
+
#define CHCR_HASH_MAX_DIGEST_SIZE 64
#define CHCR_MAX_SHA_DIGEST_SIZE 64
@@ -176,6 +209,15 @@
KEY_CONTEXT_SALT_PRESENT_V(1) | \
KEY_CONTEXT_CTX_LEN_V((ctx_len)))
+#define FILL_KEY_CRX_HDR(ck_size, mk_size, d_ck, opad, ctx_len) \
+ htonl(TLS_KEYCTX_RXMK_SIZE_V(mk_size) | \
+ TLS_KEYCTX_RXCK_SIZE_V(ck_size) | \
+ TLS_KEYCTX_RX_VALID_V(1) | \
+ TLS_KEYCTX_RX_SEQCTR_V(3) | \
+ TLS_KEYCTX_RXAUTH_MODE_V(4) | \
+ TLS_KEYCTX_RXCIPH_MODE_V(2) | \
+ TLS_KEYCTX_RXFLIT_CNT_V((ctx_len)))
+
#define FILL_WR_OP_CCTX_SIZE \
htonl( \
FW_CRYPTO_LOOKASIDE_WR_OPCODE_V( \
diff --git a/drivers/crypto/chelsio/chcr_core.h
b/drivers/crypto/chelsio/chcr_core.h
index 3c29ee0..77056a9 100644
--- a/drivers/crypto/chelsio/chcr_core.h
+++ b/drivers/crypto/chelsio/chcr_core.h
@@ -65,10 +65,58 @@
struct _key_ctx {
__be32 ctx_hdr;
u8 salt[MAX_SALT];
- __be64 reserverd;
+ __be64 iv_to_auth;
unsigned char key[0];
};
+#define KEYCTX_TX_WR_IV_S 55
+#define KEYCTX_TX_WR_IV_M 0x1ffULL
+#define KEYCTX_TX_WR_IV_V(x) ((x) << KEYCTX_TX_WR_IV_S)
+#define KEYCTX_TX_WR_IV_G(x) \
+ (((x) >> KEYCTX_TX_WR_IV_S) & KEYCTX_TX_WR_IV_M)
+
+#define KEYCTX_TX_WR_AAD_S 47
+#define KEYCTX_TX_WR_AAD_M 0xffULL
+#define KEYCTX_TX_WR_AAD_V(x) ((x) << KEYCTX_TX_WR_AAD_S)
+#define KEYCTX_TX_WR_AAD_G(x) (((x) >> KEYCTX_TX_WR_AAD_S) & \
+ KEYCTX_TX_WR_AAD_M)
+
+#define KEYCTX_TX_WR_AADST_S 39
+#define KEYCTX_TX_WR_AADST_M 0xffULL
+#define KEYCTX_TX_WR_AADST_V(x) ((x) << KEYCTX_TX_WR_AADST_S)
+#define KEYCTX_TX_WR_AADST_G(x) \
+ (((x) >> KEYCTX_TX_WR_AADST_S) & KEYCTX_TX_WR_AADST_M)
+
+#define KEYCTX_TX_WR_CIPHER_S 30
+#define KEYCTX_TX_WR_CIPHER_M 0x1ffULL
+#define KEYCTX_TX_WR_CIPHER_V(x) ((x) << KEYCTX_TX_WR_CIPHER_S)
+#define KEYCTX_TX_WR_CIPHER_G(x) \
+ (((x) >> KEYCTX_TX_WR_CIPHER_S) & KEYCTX_TX_WR_CIPHER_M)
+
+#define KEYCTX_TX_WR_CIPHERST_S 23
+#define KEYCTX_TX_WR_CIPHERST_M 0x7f
+#define KEYCTX_TX_WR_CIPHERST_V(x) ((x) << KEYCTX_TX_WR_CIPHERST_S)
+#define KEYCTX_TX_WR_CIPHERST_G(x) \
+ (((x) >> KEYCTX_TX_WR_CIPHERST_S) & KEYCTX_TX_WR_CIPHERST_M)
+
+#define KEYCTX_TX_WR_AUTH_S 14
+#define KEYCTX_TX_WR_AUTH_M 0x1ff
+#define KEYCTX_TX_WR_AUTH_V(x) ((x) << KEYCTX_TX_WR_AUTH_S)
+#define KEYCTX_TX_WR_AUTH_G(x) \
+ (((x) >> KEYCTX_TX_WR_AUTH_S) & KEYCTX_TX_WR_AUTH_M)
+
+#define KEYCTX_TX_WR_AUTHST_S 7
+#define KEYCTX_TX_WR_AUTHST_M 0x7f
+#define KEYCTX_TX_WR_AUTHST_V(x) ((x) << KEYCTX_TX_WR_AUTHST_S)
+#define KEYCTX_TX_WR_AUTHST_G(x) \
+ (((x) >> KEYCTX_TX_WR_AUTHST_S) & KEYCTX_TX_WR_AUTHST_M)
+
+#define KEYCTX_TX_WR_AUTHIN_S 0
+#define KEYCTX_TX_WR_AUTHIN_M 0x7f
+#define KEYCTX_TX_WR_AUTHIN_V(x) ((x) << KEYCTX_TX_WR_AUTHIN_S)
+#define KEYCTX_TX_WR_AUTHIN_G(x) \
+ (((x) >> KEYCTX_TX_WR_AUTHIN_S) & KEYCTX_TX_WR_AUTH
[RFC crypto v3 6/9] chtls: CPL handler definition
CPL handlers for TLS session, record transmit and receive.
Signed-off-by: Atul Gupta
---
v3: made some functions static and removed un-needed semicolon
---
drivers/crypto/chelsio/chtls/chtls_cm.c | 2045 +++
net/ipv4/tcp_minisocks.c|1 +
2 files changed, 2046 insertions(+)
create mode 100644 drivers/crypto/chelsio/chtls/chtls_cm.c
diff --git a/drivers/crypto/chelsio/chtls/chtls_cm.c
b/drivers/crypto/chelsio/chtls/chtls_cm.c
new file mode 100644
index 000..dac3c3a
--- /dev/null
+++ b/drivers/crypto/chelsio/chtls/chtls_cm.c
@@ -0,0 +1,2045 @@
+/*
+ * Copyright (c) 2017 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Written by: Atul Gupta (atul.gu...@chelsio.com)
+ */
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "chtls.h"
+#include "chtls_cm.h"
+
+extern struct request_sock_ops chtls_rsk_ops;
+
+/*
+ * State transitions and actions for close. Note that if we are in SYN_SENT
+ * we remain in that state as we cannot control a connection while it's in
+ * SYN_SENT; such connections are allowed to establish and are then aborted.
+ */
+static unsigned char new_state[16] = {
+ /* current state: new state: action: */
+ /* (Invalid) */ TCP_CLOSE,
+ /* TCP_ESTABLISHED */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
+ /* TCP_SYN_SENT*/ TCP_SYN_SENT,
+ /* TCP_SYN_RECV*/ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
+ /* TCP_FIN_WAIT1 */ TCP_FIN_WAIT1,
+ /* TCP_FIN_WAIT2 */ TCP_FIN_WAIT2,
+ /* TCP_TIME_WAIT */ TCP_CLOSE,
+ /* TCP_CLOSE */ TCP_CLOSE,
+ /* TCP_CLOSE_WAIT */ TCP_LAST_ACK | TCP_ACTION_FIN,
+ /* TCP_LAST_ACK*/ TCP_LAST_ACK,
+ /* TCP_LISTEN */ TCP_CLOSE,
+ /* TCP_CLOSING */ TCP_CLOSING,
+};
+
+static struct chtls_sock *chtls_sock_create(struct chtls_dev *cdev)
+{
+ struct chtls_sock *csk = kzalloc(sizeof(*csk), GFP_ATOMIC);
+
+ if (!csk)
+ return NULL;
+
+ csk->txdata_skb_cache = alloc_skb(TXDATA_SKB_LEN, GFP_ATOMIC);
+ if (!csk->txdata_skb_cache) {
+ kfree(csk);
+ return NULL;
+ }
+
+ kref_init(&csk->kref);
+ csk->cdev = cdev;
+ skb_queue_head_init(&csk->txq);
+ csk->wr_skb_head = NULL;
+ csk->wr_skb_tail = NULL;
+ csk->mss = MAX_MSS;
+ csk->tlshws.ofld = 1;
+ csk->tlshws.txkey = -1;
+ csk->tlshws.rxkey = -1;
+ csk->tlshws.mfs = TLS_MFS;
+ skb_queue_head_init(&csk->tlshws.sk_recv_queue);
+ return csk;
+}
+
+static void chtls_sock_release(struct kref *ref)
+{
+ struct chtls_sock *csk =
+ container_of(ref, struct chtls_sock, kref);
+
+ kfree(csk);
+}
+
+static struct net_device *chtls_ipv4_netdev(struct chtls_dev *cdev,
+ struct sock *sk)
+{
+ struct net_device *ndev = cdev->ports[0];
+
+ if (likely(!inet_sk(sk)->inet_rcv_saddr))
+ return ndev;
+
+ ndev = ip_dev_find(&init_net, inet_sk(sk)->inet_rcv_saddr);
+ if (!ndev)
+ return NULL;
+
+ if (is_vlan_dev(ndev))
+ return vlan_dev_real_dev(ndev);
+ return ndev;
+}
+
+static void assign_rxopt(struct sock *sk, unsigned int opt)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+ struct tcp_sock *tp = tcp_sk(sk);
+ const struct chtls_dev *cdev;
+
+ cdev = csk->cdev;
+ tp->tcp_header_len = sizeof(struct tcphdr);
+ tp->rx_opt.mss_clamp = cdev->mtus[TCPOPT_MSS_G(opt)] - 40;
+ tp->mss_cache= tp->rx_opt.mss_clamp;
+ tp->rx_opt.tstamp_ok = TCPOPT_TSTAMP_G(opt);
+ tp->rx_opt.snd_wscale= TCPOPT_SACK_G(opt);
+ tp->rx_opt.wscale_ok = TCPOPT_WSCALE_OK_G(opt);
+ SND_WSCALE(tp) = TCPOPT_SND_WSCALE_G(opt);
+ if (!tp->rx_opt.wscale_ok)
+ tp->rx_opt.rcv_wscale = 0;
+ if (tp->rx_opt.tstamp_ok) {
+ tp->tcp_header_len += TCPOLEN_TSTAMP_ALIGNED;
+ tp->rx_opt.mss_clamp -= TCPOLEN_TSTAMP_ALIGNED;
+ } else if (csk->opt2 & TSTAMPS_EN_F) {
+ csk->opt2 &= ~TSTAMPS_EN_F;
+ csk->mtu_idx = TCPOPT_MSS_G(opt);
+ }
+}
+
+static void chtls_purge_rcv_queue(struct sock *sk)
+{
+ struct sk_buff *skb;
+
+ while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) {
+ skb_dst_set(skb, (void *)NULL);
+ kfree_skb(skb)
[RFC crypto v3 7/9] chtls: Inline crypto request Tx/Rx
TLS handler for record transmit and receive.
Create Inline TLS work request and post to FW.
Signed-off-by: Atul Gupta
---
v3: made some functions static and initialized few variables
---
drivers/crypto/chelsio/chtls/chtls_io.c | 1867 +++
1 file changed, 1867 insertions(+)
create mode 100644 drivers/crypto/chelsio/chtls/chtls_io.c
diff --git a/drivers/crypto/chelsio/chtls/chtls_io.c
b/drivers/crypto/chelsio/chtls/chtls_io.c
new file mode 100644
index 000..a0f03fb
--- /dev/null
+++ b/drivers/crypto/chelsio/chtls/chtls_io.c
@@ -0,0 +1,1867 @@
+/*
+ * Copyright (c) 2017 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Written by: Atul Gupta (atul.gu...@chelsio.com)
+ */
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "chtls.h"
+#include "chtls_cm.h"
+
+static bool is_tls_hw(struct chtls_sock *csk)
+{
+ return csk->tlshws.ofld;
+}
+
+static bool is_tls_rx(struct chtls_sock *csk)
+{
+ return (csk->tlshws.rxkey >= 0);
+}
+
+static bool is_tls_tx(struct chtls_sock *csk)
+{
+ return (csk->tlshws.txkey >= 0);
+}
+
+static bool is_tls_skb(struct chtls_sock *csk, const struct sk_buff *skb)
+{
+ return (is_tls_hw(csk) && skb_ulp_tls_skb_flags(skb));
+}
+
+static int key_size(void *sk)
+{
+ return 16; /* Key on DDR */
+}
+
+#define ceil(x, y) \
+ ({ unsigned long __x = (x), __y = (y); (__x + __y - 1) / __y; })
+
+static int data_sgl_len(const struct sk_buff *skb)
+{
+ unsigned int cnt;
+
+ cnt = skb_shinfo(skb)->nr_frags;
+ return (sgl_len(cnt) * 8);
+}
+
+static int nos_ivs(struct sock *sk, unsigned int size)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+
+ return ceil(size, csk->tlshws.mfs);
+}
+
+#define TLS_WR_CPL_LEN \
+ (sizeof(struct fw_tlstx_data_wr) + \
+ sizeof(struct cpl_tx_tls_sfo))
+
+static int is_ivs_imm(struct sock *sk, const struct sk_buff *skb)
+{
+ int ivs_size = nos_ivs(sk, skb->len) * CIPHER_BLOCK_SIZE;
+ int hlen = TLS_WR_CPL_LEN + data_sgl_len(skb);
+
+ if ((hlen + key_size(sk) + ivs_size) <
+ MAX_IMM_OFLD_TX_DATA_WR_LEN) {
+ ULP_SKB_CB(skb)->ulp.tls.iv = 1;
+ return 1;
+ }
+ ULP_SKB_CB(skb)->ulp.tls.iv = 0;
+ return 0;
+}
+
+static int max_ivs_size(struct sock *sk, int size)
+{
+ return (nos_ivs(sk, size) * CIPHER_BLOCK_SIZE);
+}
+
+static int ivs_size(struct sock *sk, const struct sk_buff *skb)
+{
+ return (is_ivs_imm(sk, skb) ? (nos_ivs(sk, skb->len) *
+CIPHER_BLOCK_SIZE) : 0);
+}
+
+static int flowc_wr_credits(int nparams, int *flowclenp)
+{
+ int flowclen16, flowclen;
+
+ flowclen = offsetof(struct fw_flowc_wr, mnemval[nparams]);
+ flowclen16 = DIV_ROUND_UP(flowclen, 16);
+ flowclen = flowclen16 * 16;
+
+ if (flowclenp)
+ *flowclenp = flowclen;
+
+ return flowclen16;
+}
+
+static struct sk_buff *create_flowc_wr_skb(struct sock *sk,
+ struct fw_flowc_wr *flowc,
+ int flowclen)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+ struct sk_buff *skb;
+
+ skb = alloc_skb(flowclen, GFP_ATOMIC);
+ if (!skb)
+ return NULL;
+
+ memcpy(__skb_put(skb, flowclen), flowc, flowclen);
+ set_queue(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA, sk);
+
+ return skb;
+}
+
+static int send_flowc_wr(struct sock *sk, struct fw_flowc_wr *flowc,
+int flowclen)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+ bool syn_sent = (sk->sk_state == TCP_SYN_SENT);
+ struct tcp_sock *tp = tcp_sk(sk);
+ int flowclen16 = flowclen / 16;
+ struct sk_buff *skb;
+
+ if (csk_flag(sk, CSK_TX_DATA_SENT)) {
+ skb = create_flowc_wr_skb(sk, flowc, flowclen);
+ if (!skb)
+ return -ENOMEM;
+
+ if (syn_sent)
+ __skb_queue_tail(&csk->ooo_queue, skb);
+ else
+ skb_entail(sk, skb,
+ ULPCB_FLAG_NO_HDR | ULPCB_FLAG_NO_APPEND);
+ return 0;
+ }
+
+ if (!syn_sent) {
+ int ret;
+
+ ret = cxgb4_immdata_send(csk->egress_dev,
+csk->txq_idx,
+flowc, flowclen);
+ if (!ret)
+ return flowclen16;
+ }
+ skb = create_flowc_wr_skb(sk, flowc, flowclen
[RFC crypto v3 8/9] chtls: Register the ULP
Add new uld driver for Inline TLS support. Register ULP for chtls.
Setsockopt to program key on chip. support AES GCM key size 128.
Signed-off-by: Atul Gupta
---
v3: made some functions static
---
drivers/crypto/chelsio/chtls/chtls_main.c | 584 ++
include/uapi/linux/tls.h | 1 +
2 files changed, 585 insertions(+)
create mode 100644 drivers/crypto/chelsio/chtls/chtls_main.c
diff --git a/drivers/crypto/chelsio/chtls/chtls_main.c
b/drivers/crypto/chelsio/chtls/chtls_main.c
new file mode 100644
index 000..fb2d441
--- /dev/null
+++ b/drivers/crypto/chelsio/chtls/chtls_main.c
@@ -0,0 +1,584 @@
+/*
+ * Copyright (c) 2017 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Written by: Atul Gupta (atul.gu...@chelsio.com)
+ */
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "chtls.h"
+#include "chtls_cm.h"
+
+#define DRV_NAME "chtls"
+
+/*
+ * chtls device management
+ * maintains a list of the chtls devices
+ */
+static LIST_HEAD(cdev_list);
+static DEFINE_MUTEX(cdev_mutex);
+static DEFINE_MUTEX(cdev_list_lock);
+
+static struct proto chtls_base_prot;
+static struct proto chtls_cpl_prot;
+static DEFINE_MUTEX(notify_mutex);
+static RAW_NOTIFIER_HEAD(listen_notify_list);
+struct request_sock_ops chtls_rsk_ops;
+static uint send_page_order = (14 - PAGE_SHIFT < 0) ? 0 : 14 - PAGE_SHIFT;
+
+static int register_listen_notifier(struct notifier_block *nb)
+{
+ int err;
+
+ mutex_lock(¬ify_mutex);
+ err = raw_notifier_chain_register(&listen_notify_list, nb);
+ mutex_unlock(¬ify_mutex);
+ return err;
+}
+
+static int unregister_listen_notifier(struct notifier_block *nb)
+{
+ int err;
+
+ mutex_lock(¬ify_mutex);
+ err = raw_notifier_chain_unregister(&listen_notify_list, nb);
+ mutex_unlock(¬ify_mutex);
+ return err;
+}
+
+static int listen_notify_handler(struct notifier_block *this,
+unsigned long event, void *data)
+{
+ struct sock *sk = data;
+ struct chtls_dev *cdev;
+ int ret = NOTIFY_DONE;
+
+ switch (event) {
+ case CHTLS_LISTEN_START:
+ case CHTLS_LISTEN_STOP:
+ mutex_lock(&cdev_list_lock);
+ list_for_each_entry(cdev, &cdev_list, list) {
+ if (event == CHTLS_LISTEN_START)
+ ret = chtls_listen_start(cdev, sk);
+ else
+ chtls_listen_stop(cdev, sk);
+ }
+ mutex_unlock(&cdev_list_lock);
+ break;
+ }
+ return ret;
+}
+
+static struct notifier_block listen_notifier = {
+ .notifier_call = listen_notify_handler
+};
+
+static int listen_backlog_rcv(struct sock *sk, struct sk_buff *skb)
+{
+ if (likely(skb_transport_header(skb) != skb_network_header(skb)))
+ return tcp_v4_do_rcv(sk, skb);
+ BLOG_SKB_CB(skb)->backlog_rcv(sk, skb);
+ return 0;
+}
+
+static int chtls_start_listen(struct sock *sk)
+{
+ int err;
+
+ if (sk->sk_protocol != IPPROTO_TCP)
+ return -EPROTONOSUPPORT;
+
+ if (sk->sk_family == PF_INET &&
+ LOOPBACK(inet_sk(sk)->inet_rcv_saddr))
+ return -EADDRNOTAVAIL;
+
+ sk->sk_backlog_rcv = listen_backlog_rcv;
+ mutex_lock(¬ify_mutex);
+ err = raw_notifier_call_chain(&listen_notify_list, 0, sk);
+ mutex_unlock(¬ify_mutex);
+ return err;
+}
+
+static int chtls_hash(struct sock *sk)
+{
+ int err;
+
+ err = tcp_prot.hash(sk);
+ if (sk->sk_state == TCP_LISTEN)
+ err |= chtls_start_listen(sk);
+
+ if (err)
+ tcp_prot.unhash(sk);
+ return err;
+}
+
+static int chtls_stop_listen(struct sock *sk)
+{
+ if (sk->sk_protocol != IPPROTO_TCP)
+ return -EPROTONOSUPPORT;
+
+ mutex_lock(¬ify_mutex);
+ raw_notifier_call_chain(&listen_notify_list, 1, sk);
+ mutex_unlock(¬ify_mutex);
+ return 0;
+}
+
+static void chtls_unhash(struct sock *sk)
+{
+ if (sk->sk_state == TCP_LISTEN)
+ chtls_stop_listen(sk);
+ tcp_prot.unhash(sk);
+}
+
+static void chtls_lsk_close(struct sock *sk, long timeout)
+{
+ struct tls_context *ctx = tls_get_ctx(sk);
+ void (*sk_proto_close)(struct sock *sk, long timeout);
+
+ lock_sock(sk);
+ sk_proto_close = ctx->sk_proto_close;
+ kfree(ctx);
+
+ release_sock(sk);
+ sk_proto_close(sk, timeout);
+}
+
+static void process_deferq(struct work_struct *task_param)
+{
+ struct chtls_dev *cdev = container_of(ta
[RFC crypto v3 9/9] Makefile Kconfig
Entry for Inline TLS as another driver dependent on cxgb4 and chcr Signed-off-by: Atul Gupta --- drivers/crypto/chelsio/Kconfig| 10 ++ drivers/crypto/chelsio/Makefile | 1 + drivers/crypto/chelsio/chtls/Makefile | 4 3 files changed, 15 insertions(+) create mode 100644 drivers/crypto/chelsio/chtls/Makefile diff --git a/drivers/crypto/chelsio/Kconfig b/drivers/crypto/chelsio/Kconfig index 51932c7..686d246 100644 --- a/drivers/crypto/chelsio/Kconfig +++ b/drivers/crypto/chelsio/Kconfig @@ -28,3 +28,13 @@ config CHELSIO_IPSEC_INLINE default n ---help--- Enable support for IPSec Tx Inline. + +config CRYPTO_DEV_CHELSIO_TLS +tristate "Chelsio Crypto Inline TLS Driver" +depends on CHELSIO_T4 +select CRYPTO_DEV_CHELSIO +---help--- + Support Chelsio Inline TLS with Chelsio crypto accelerator. + + To compile this driver as a module, choose M here: the module + will be called chtls. diff --git a/drivers/crypto/chelsio/Makefile b/drivers/crypto/chelsio/Makefile index eaecaf1..639e571 100644 --- a/drivers/crypto/chelsio/Makefile +++ b/drivers/crypto/chelsio/Makefile @@ -3,3 +3,4 @@ ccflags-y := -Idrivers/net/ethernet/chelsio/cxgb4 obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chcr.o chcr-objs := chcr_core.o chcr_algo.o chcr-$(CONFIG_CHELSIO_IPSEC_INLINE) += chcr_ipsec.o +obj-$(CONFIG_CRYPTO_DEV_CHELSIO_TLS) += chtls/ diff --git a/drivers/crypto/chelsio/chtls/Makefile b/drivers/crypto/chelsio/chtls/Makefile new file mode 100644 index 000..df13795 --- /dev/null +++ b/drivers/crypto/chelsio/chtls/Makefile @@ -0,0 +1,4 @@ +ccflags-y := -Idrivers/net/ethernet/chelsio/cxgb4 -Idrivers/crypto/chelsio/ + +obj-$(CONFIG_CRYPTO_DEV_CHELSIO_TLS) += chtls.o +chtls-objs := chtls_main.o chtls_cm.o chtls_io.o chtls_hw.o -- 1.8.3.1
[Crypto chcr] crypto: af_alg - cast ki_complete call's ternary operator variables to long.
The ki_complete called from af_alg_async_cb use ternary operator to get
the value of second argument.As err is signed int while resultlen is
unsigned int, by the precedence rule err is also processed as unsigned
int and lose its original value.Hence, it is advised to cast both err
and resultlen as long which is expected by the definition of ki_complete
call as its 2nd argument. This will retain the original signed value of
err.
Declaration of ki_complete in file linux/include/linux/fs.h in struct
kiocb {...
void (*ki_complete)(struct kiocb *iocb, long ret, long ret2);
...
}
Signed-off-by: Atul Gupta
---
crypto/af_alg.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/crypto/af_alg.c b/crypto/af_alg.c
index edca099..8e48d97 100644
--- a/crypto/af_alg.c
+++ b/crypto/af_alg.c
@@ -1048,7 +1048,7 @@ void af_alg_async_cb(struct crypto_async_request *_req,
int err)
af_alg_free_resources(areq);
sock_put(sk);
- iocb->ki_complete(iocb, err ? err : resultlen, 0);
+ iocb->ki_complete(iocb, err ? (long)err : (long)resultlen, 0);
}
EXPORT_SYMBOL_GPL(af_alg_async_cb);
--
1.8.3.1
Re: [RFC crypto v3 0/9] Chelsio Inline TLS
Addressed the review comments in v2 and v3, please suggest if there is
any other comment and step to proceed?
Thanks
Atul Gupta
On Wednesday 20 December 2017 05:03 PM, Atul Gupta wrote:
RFC series for Chelsio Inline TLS driver (chtls.ko)
Driver use the ULP infrastructure to register chtls as Inline TLS ULP.
Chtls use TCP Sockets to transmit and receive TLS record. TCP proto_ops
is extended to offload TLS record.
T6 adapter provides the following features:
-TLS record offload, TLS header, encrypt, digest and transmit
-TLS record receive and decrypt
-TLS keys store
-TCP/IP engine
-TLS engine
-GCM crypto engine [support CBC also]
TLS provides security at the transport layer. It uses TCP to provide
reliable end-to-end transport of application data. It relies on TCP
for any retransmission. TLS session comprises of three parts:
a. TCP/IP connection
b. TLS handshake
c. Record layer processing
TLS handshake state machine is executed in host (refer standard
implementation eg. OpenSSL). Setsockopt [SOL_TCP, TCP_ULP] initialize
TCP proto-ops for Chelsio inline tls support. setsockopt(sock, SOL_TCP,
TCP_ULP, "chtls", sizeof("chtls"));
Tx and Rx Keys are decided during handshake and programmed onto the chip
after CCS is exchanged.
struct tls12_crypto_info_aes_gcm_128 crypto_info
setsockopt(sock, SOL_TLS, TLS_TX, &crypto_info, sizeof(crypto_info))
Finish is the first encrypted/decrypted message tx/rx inline.
On the Tx path TLS engine receive plain text from openssl, insert IV,
fetches the tx key, create cipher text records and generate MAC. TLS
header is added to cipher text and forward to TCP/IP engine for transport
layer processing and transmission on wire.
TX:
Application--openssl--chtls---TLS engine---encrypt/auth---TCP/IP
engine---wire.
On the Rx side, data received is PDU aligned at record
boundaries. TLS processes only the complete record. If rx key is programmed
on CCS receive, data is decrypted and plain text is posted to host.
RX:
Wire--cipher-text--TCP/IP engine [PDU align]---TLS engine---
decrypt/auth---plain-text--chtls--openssl--application
v3: fixed the kbuild test issues
-made few funtions static
-initialized few variables
v2: fixed the following based on the review comments of Stephan Mueller,
Stefano Brivio and Hannes Frederic
-Added more details in cover letter
-Fixed indentation and formating issues
-Using aes instead of aes-generic
-memset key info after programing the key on chip
-reordered the patch sequence
Atul Gupta (9):
chtls: structure and macro definiton
cxgb4: Inline TLS FW Interface
cxgb4: LLD driver changes to enable TLS
chcr: Key Macro
chtls: Key program
chtls: CPL handler definition
chtls: Inline crypto request Tx/Rx
chtls: Register the ULP
Makefile Kconfig
drivers/crypto/chelsio/Kconfig | 10 +
drivers/crypto/chelsio/Makefile|1 +
drivers/crypto/chelsio/chcr_algo.h | 42 +
drivers/crypto/chelsio/chcr_core.h | 55 +-
drivers/crypto/chelsio/chtls/Makefile |4 +
drivers/crypto/chelsio/chtls/chtls.h | 480 +
drivers/crypto/chelsio/chtls/chtls_cm.c| 2045
drivers/crypto/chelsio/chtls/chtls_cm.h| 203 ++
drivers/crypto/chelsio/chtls/chtls_hw.c| 394
drivers/crypto/chelsio/chtls/chtls_io.c| 1867 ++
drivers/crypto/chelsio/chtls/chtls_main.c | 584 ++
drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c | 18 +-
drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c| 32 +-
drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h |7 +
drivers/net/ethernet/chelsio/cxgb4/sge.c | 98 +-
drivers/net/ethernet/chelsio/cxgb4/t4_msg.h| 121 +-
drivers/net/ethernet/chelsio/cxgb4/t4_regs.h |2 +
drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h | 165 +-
include/uapi/linux/tls.h |1 +
net/ipv4/tcp_minisocks.c |1 +
20 files changed, 6111 insertions(+), 19 deletions(-)
create mode 100644 drivers/crypto/chelsio/chtls/Makefile
create mode 100644 drivers/crypto/chelsio/chtls/chtls.h
create mode 100644 drivers/crypto/chelsio/chtls/chtls_cm.c
create mode 100644 drivers/crypto/chelsio/chtls/chtls_cm.h
create mode 100644 drivers/crypto/chelsio/chtls/chtls_hw.c
create mode 100644 drivers/crypto/chelsio/chtls/chtls_io.c
create mode 100644 drivers/crypto/chelsio/chtls/chtls_main.c
RE: [RFC crypto v3 0/9] Chelsio Inline TLS
Would truly appreciate your feedback and make progress.
Thanks and Regards
Atul
-Original Message-
From: linux-crypto-ow...@vger.kernel.org
[mailto:linux-crypto-ow...@vger.kernel.org] On Behalf Of Atul Gupta
Sent: Wednesday, January 3, 2018 12:36 PM
To: herb...@gondor.apana.org.au; linux-crypto@vger.kernel.org; Ganesh GR
Cc: net...@vger.kernel.org; da...@davemloft.net; davejwat...@fb.com;
smuel...@chronox.de; Stefano Brivio ;
han...@stressinduktion.org
Subject: Re: [RFC crypto v3 0/9] Chelsio Inline TLS
Addressed the review comments in v2 and v3, please suggest if there is any
other comment and step to proceed?
Thanks
Atul Gupta
On Wednesday 20 December 2017 05:03 PM, Atul Gupta wrote:
> RFC series for Chelsio Inline TLS driver (chtls.ko)
>
> Driver use the ULP infrastructure to register chtls as Inline TLS ULP.
> Chtls use TCP Sockets to transmit and receive TLS record. TCP
> proto_ops is extended to offload TLS record.
>
> T6 adapter provides the following features:
> -TLS record offload, TLS header, encrypt, digest and transmit
> -TLS record receive and decrypt
> -TLS keys store
> -TCP/IP engine
> -TLS engine
> -GCM crypto engine [support CBC also]
>
> TLS provides security at the transport layer. It uses TCP to provide
> reliable end-to-end transport of application data. It relies on TCP
> for any retransmission. TLS session comprises of three parts:
> a. TCP/IP connection
> b. TLS handshake
> c. Record layer processing
>
> TLS handshake state machine is executed in host (refer standard
> implementation eg. OpenSSL). Setsockopt [SOL_TCP, TCP_ULP] initialize
> TCP proto-ops for Chelsio inline tls support. setsockopt(sock,
> SOL_TCP, TCP_ULP, "chtls", sizeof("chtls"));
>
> Tx and Rx Keys are decided during handshake and programmed onto the
> chip after CCS is exchanged.
> struct tls12_crypto_info_aes_gcm_128 crypto_info setsockopt(sock,
> SOL_TLS, TLS_TX, &crypto_info, sizeof(crypto_info)) Finish is the
> first encrypted/decrypted message tx/rx inline.
>
> On the Tx path TLS engine receive plain text from openssl, insert IV,
> fetches the tx key, create cipher text records and generate MAC. TLS
> header is added to cipher text and forward to TCP/IP engine for
> transport layer processing and transmission on wire.
> TX:
> Application--openssl--chtls---TLS engine---encrypt/auth---TCP/IP
> engine---wire.
>
> On the Rx side, data received is PDU aligned at record boundaries. TLS
> processes only the complete record. If rx key is programmed on CCS
> receive, data is decrypted and plain text is posted to host.
> RX:
> Wire--cipher-text--TCP/IP engine [PDU align]---TLS engine---
> decrypt/auth---plain-text--chtls--openssl--application
>
> v3: fixed the kbuild test issues
> -made few funtions static
> -initialized few variables
>
> v2: fixed the following based on the review comments of Stephan Mueller,
> Stefano Brivio and Hannes Frederic
> -Added more details in cover letter
> -Fixed indentation and formating issues
> -Using aes instead of aes-generic
> -memset key info after programing the key on chip
> -reordered the patch sequence
>
> Atul Gupta (9):
>chtls: structure and macro definiton
>cxgb4: Inline TLS FW Interface
>cxgb4: LLD driver changes to enable TLS
>chcr: Key Macro
>chtls: Key program
>chtls: CPL handler definition
>chtls: Inline crypto request Tx/Rx
>chtls: Register the ULP
>Makefile Kconfig
>
> drivers/crypto/chelsio/Kconfig | 10 +
> drivers/crypto/chelsio/Makefile|1 +
> drivers/crypto/chelsio/chcr_algo.h | 42 +
> drivers/crypto/chelsio/chcr_core.h | 55 +-
> drivers/crypto/chelsio/chtls/Makefile |4 +
> drivers/crypto/chelsio/chtls/chtls.h | 480 +
> drivers/crypto/chelsio/chtls/chtls_cm.c| 2045
>
> drivers/crypto/chelsio/chtls/chtls_cm.h| 203 ++
> drivers/crypto/chelsio/chtls/chtls_hw.c| 394
> drivers/crypto/chelsio/chtls/chtls_io.c| 1867 ++
> drivers/crypto/chelsio/chtls/chtls_main.c | 584 ++
> drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c | 18 +-
> drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c| 32 +-
> drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h |7 +
> drivers/net/ethernet/chelsio/cxgb4/sge.c | 98 +-
> drivers/net/ethernet/chelsio/cxgb4/t4_msg.h| 121 +-
> drivers/net/ethernet/chelsio/cxgb4/t4_regs.h |2 +
> drivers/net/ethe
Re: [RFC crypto v3 0/9] Chelsio Inline TLS
On Monday 22 January 2018 03:46 AM, Sabrina Dubroca wrote: 2017-12-20, 17:03:02 +0530, Atul Gupta wrote: RFC series for Chelsio Inline TLS driver (chtls.ko) Driver use the ULP infrastructure to register chtls as Inline TLS ULP. I don't think drivers should be registering their own ULP. TLS offloading should be transparent to userspace, whatever HW ends up being used. If each driver implements its own ULP, the application has to be aware of what HW and what driver it's running on. using different ULP is derived from https://patchwork.kernel.org/patch/9746381/ I think this offload should rely on a generic infrastructure, not build its own private interface. Look at the current kTLS code, the proposal for an offload infrastructure [0] from Mellanox, and see how you can fit your driver into that, and extend what's missing. [0] https://patchwork.ozlabs.org/patch/849984/ The driver indeed used the proposed offload infrastructure and extended for Inline Rx/Tx [...] Atul Gupta (9): chtls: structure and macro definiton cxgb4: Inline TLS FW Interface cxgb4: LLD driver changes to enable TLS chcr: Key Macro chtls: Key program chtls: CPL handler definition chtls: Inline crypto request Tx/Rx chtls: Register the ULP Makefile Kconfig That patchset is split so that each patch touches a separate set of files, and the description of the contents of each patch is very limited. Can you try to group your changes by feature instead? That should help you come up with descriptive commit messages as well. Made all attempts to group the contents based on functionality, the set is broken into driver registration, I/O with crypto Inline request, key handling and messages exchanged with hardware. Thanks,
RE: [RFC crypto v3 8/9] chtls: Register the ULP
-Original Message-
From: Dave Watson [mailto:davejwat...@fb.com]
Sent: Friday, January 26, 2018 2:39 AM
To: Atul Gupta
Cc: herb...@gondor.apana.org.au; linux-crypto@vger.kernel.org;
ganes...@chelsio.co; net...@vger.kernel.org; da...@davemloft.net; Boris
Pismenny ; Ilya Lesokhin
Subject: Re: [RFC crypto v3 8/9] chtls: Register the ULP
<1513769897-26945-1-git-send-email-atul.gu...@chelsio.com>
On 12/20/17 05:08 PM, Atul Gupta wrote:
> +static void __init chtls_init_ulp_ops(void) {
> + chtls_base_prot = tcp_prot;
> + chtls_base_prot.hash= chtls_hash;
> + chtls_base_prot.unhash = chtls_unhash;
> + chtls_base_prot.close = chtls_lsk_close;
> +
> + chtls_cpl_prot = chtls_base_prot;
> + chtls_init_rsk_ops(&chtls_cpl_prot, &chtls_rsk_ops,
> +&tcp_prot, PF_INET);
> + chtls_cpl_prot.close= chtls_close;
> + chtls_cpl_prot.disconnect = chtls_disconnect;
> + chtls_cpl_prot.destroy = chtls_destroy_sock;
> + chtls_cpl_prot.shutdown = chtls_shutdown;
> + chtls_cpl_prot.sendmsg = chtls_sendmsg;
> + chtls_cpl_prot.recvmsg = chtls_recvmsg;
> + chtls_cpl_prot.sendpage = chtls_sendpage;
> + chtls_cpl_prot.setsockopt = chtls_setsockopt;
> + chtls_cpl_prot.getsockopt = chtls_getsockopt;
> +}
Much of this file should go in tls_main.c, reusing as much as possible. For
example it doesn't look like the get/set sockopts have changed at all for chtls.
Agree, should common code and anything other than TLS_BASE_TX/TLS_SW_TX prot
should go in vendor specific file/driver. Since, prot require redefinition for
hardware the code is kept in chtls_main.c
> +
> +static int __init chtls_register(void) {
> + chtls_init_ulp_ops();
> + register_listen_notifier(&listen_notifier);
> + cxgb4_register_uld(CXGB4_ULD_TLS, &chtls_uld_info);
> + tcp_register_ulp(&tcp_chtls_ulp_ops);
> + return 0;
> +}
> +
> +static void __exit chtls_unregister(void) {
> + unregister_listen_notifier(&listen_notifier);
> + tcp_unregister_ulp(&tcp_chtls_ulp_ops);
> + chtls_free_all_uld();
> + cxgb4_unregister_uld(CXGB4_ULD_TLS);
> +}
The idea with ULP is that there is one ULP hook per protocol, not per driver.
One thought is that apps/lib calling setsockopt pass the required ulp type [tls
or chtls or xtls], this enables any HW assist to define base_prot as required
and keep common code [tls_main] independent of underlying HW.
If we are to have single TLS ULP hook [good from user point] then need a way to
determine which Inline tls hw is used? System with multiple Inline TLS capable
hw and differing functionality would require checks in tls_main to exercise
that specific functionality/callback?
RE: [RFC crypto v3 8/9] chtls: Register the ULP
@Dave Watson, Did you get chance to look at my response?
What I was referring is that passing "tls" ulp type in setsockopt may be
insufficient to make the decision when multi HW assist Inline TLS solution
exists.
Some HW may go beyond defining sendmsg/sendpage of the prot and require
additional info to setup the env? Also, we need to keep vendor specific code
out of tls_main.c i.e anything other than base/sw_tx prot perhaps go to hw
driver.
Sabrina echoed similar concern early last week, can we discuss or have thoughts
how to address this?
Thanks
Atul
-Original Message-----
From: Atul Gupta
Sent: Sunday, January 28, 2018 11:26 AM
To: 'Dave Watson'
Cc: herb...@gondor.apana.org.au; linux-crypto@vger.kernel.org;
ganes...@chelsio.co; net...@vger.kernel.org; da...@davemloft.net; Boris
Pismenny ; Ilya Lesokhin
Subject: RE: [RFC crypto v3 8/9] chtls: Register the ULP
-Original Message-
From: Dave Watson [mailto:davejwat...@fb.com]
Sent: Friday, January 26, 2018 2:39 AM
To: Atul Gupta
Cc: herb...@gondor.apana.org.au; linux-crypto@vger.kernel.org;
ganes...@chelsio.co; net...@vger.kernel.org; da...@davemloft.net; Boris
Pismenny ; Ilya Lesokhin
Subject: Re: [RFC crypto v3 8/9] chtls: Register the ULP
<1513769897-26945-1-git-send-email-atul.gu...@chelsio.com>
On 12/20/17 05:08 PM, Atul Gupta wrote:
> +static void __init chtls_init_ulp_ops(void) {
> + chtls_base_prot = tcp_prot;
> + chtls_base_prot.hash= chtls_hash;
> + chtls_base_prot.unhash = chtls_unhash;
> + chtls_base_prot.close = chtls_lsk_close;
> +
> + chtls_cpl_prot = chtls_base_prot;
> + chtls_init_rsk_ops(&chtls_cpl_prot, &chtls_rsk_ops,
> +&tcp_prot, PF_INET);
> + chtls_cpl_prot.close= chtls_close;
> + chtls_cpl_prot.disconnect = chtls_disconnect;
> + chtls_cpl_prot.destroy = chtls_destroy_sock;
> + chtls_cpl_prot.shutdown = chtls_shutdown;
> + chtls_cpl_prot.sendmsg = chtls_sendmsg;
> + chtls_cpl_prot.recvmsg = chtls_recvmsg;
> + chtls_cpl_prot.sendpage = chtls_sendpage;
> + chtls_cpl_prot.setsockopt = chtls_setsockopt;
> + chtls_cpl_prot.getsockopt = chtls_getsockopt;
> +}
Much of this file should go in tls_main.c, reusing as much as possible. For
example it doesn't look like the get/set sockopts have changed at all for chtls.
Agree, should common code and anything other than TLS_BASE_TX/TLS_SW_TX prot
should go in vendor specific file/driver. Since, prot require redefinition for
hardware the code is kept in chtls_main.c
> +
> +static int __init chtls_register(void) {
> + chtls_init_ulp_ops();
> + register_listen_notifier(&listen_notifier);
> + cxgb4_register_uld(CXGB4_ULD_TLS, &chtls_uld_info);
> + tcp_register_ulp(&tcp_chtls_ulp_ops);
> + return 0;
> +}
> +
> +static void __exit chtls_unregister(void) {
> + unregister_listen_notifier(&listen_notifier);
> + tcp_unregister_ulp(&tcp_chtls_ulp_ops);
> + chtls_free_all_uld();
> + cxgb4_unregister_uld(CXGB4_ULD_TLS);
> +}
The idea with ULP is that there is one ULP hook per protocol, not per driver.
One thought is that apps/lib calling setsockopt pass the required ulp type [tls
or chtls or xtls], this enables any HW assist to define base_prot as required
and keep common code [tls_main] independent of underlying HW.
If we are to have single TLS ULP hook [good from user point] then need a way to
determine which Inline tls hw is used? System with multiple Inline TLS capable
hw and differing functionality would require checks in tls_main to exercise
that specific functionality/callback?
Re: [RFC crypto v3 8/9] chtls: Register the ULP
On Tuesday 30 January 2018 10:41 PM, Dave Watson wrote:
On 01/30/18 06:51 AM, Atul Gupta wrote:
What I was referring is that passing "tls" ulp type in setsockopt
may be insufficient to make the decision when multi HW assist Inline
TLS solution exists.
Setting the ULP doesn't choose HW or SW implementation, I think that
should be done later when setting up crypto with
setsockopt(SOL_TLS, TLS_TX, struct crypto_info).
setsockpot [mentioned above] is quite late for driver to enable HW
implementation, we require something as early as tls_init
[setsockopt(sock, SOL_TCP, TCP_ULP, "tls", sizeof("tls"))], for driver
to set HW prot and offload connection beside Inline Tx/Rx.
Any reason we can't use ethtool to choose HW vs SW implementation, if
available on the device?
Thought about it, the interface index is not available to fetch netdev
and caps check to set HW prot eg. bind [prot.hash] --> tls_hash to
program HW.
Some HW may go beyond defining sendmsg/sendpage of the prot and
require additional info to setup the env? Also, we need to keep
vendor specific code out of tls_main.c i.e anything other than
base/sw_tx prot perhaps go to hw driver.
Sure, but I think we can add hooks to tls_main to do this without a
new ULP.
Current code calls update_sk_prot for TLS_BASE_TX and TLS_SW_TX, future
Inline TLS assist HWs will add TLS_HW_TX, TLS_OFLD, ... etc additional
hooks to update sk prots can make code confusing?
RE: [RFC crypto v3 8/9] chtls: Register the ULP
-Original Message-
From: Dave Watson [mailto:davejwat...@fb.com]
Sent: Wednesday, January 31, 2018 10:14 PM
To: Atul Gupta
Cc: s...@queasysnail.net; herb...@gondor.apana.org.au;
linux-crypto@vger.kernel.org; ganes...@chelsio.co; net...@vger.kernel.org;
da...@davemloft.net; Boris Pismenny ; Ilya Lesokhin
Subject: Re: [RFC crypto v3 8/9] chtls: Register the ULP
On 01/31/18 04:14 PM, Atul Gupta wrote:
>
>
> On Tuesday 30 January 2018 10:41 PM, Dave Watson wrote:
> > On 01/30/18 06:51 AM, Atul Gupta wrote:
> >
> > > What I was referring is that passing "tls" ulp type in setsockopt
> > > may be insufficient to make the decision when multi HW assist
> > > Inline TLS solution exists.
> > Setting the ULP doesn't choose HW or SW implementation, I think that
> > should be done later when setting up crypto with
> >
> > setsockopt(SOL_TLS, TLS_TX, struct crypto_info).
> setsockpot [mentioned above] is quite late for driver to enable HW
> implementation, we require something as early as tls_init
> [setsockopt(sock, SOL_TCP, TCP_ULP, "tls", sizeof("tls"))], for driver
> to set HW prot and offload connection beside Inline Tx/Rx.
> >
> > Any reason we can't use ethtool to choose HW vs SW implementation,
> > if available on the device?
> Thought about it, the interface index is not available to fetch
> netdev and caps check to set HW prot eg. bind [prot.hash] --> tls_hash to
> program HW.
Perhaps this is the part I don't follow - why do you need to override hash and
check for LISTEN? I briefly looked through the patch named "CPL handler
definition", this looks like it is a full TCP offload?
Yes, this is connection and record layer offload, and the reason I used
different ulp type, need to see what additional info or check can help setup
the required sk prot.
RE: [RFC crypto v3 8/9] chtls: Register the ULP
I thought about this and approach below can avoid new ulp type:
1. Register Inline TLS driver to net TLS
2. enable ethtool -K tls-hw-record-offload on
3. Issue " setsockopt(fd, SOL_TCP, TCP_ULP, "tls", sizeof("tls")) " after Bind,
this will enable user fetch net_device corresponding to ipaadr bound to
interface, if dev found is the one registered and record-offload enabled,
program the sk->sk_prot as required.
4. fallback to SW TLS for any other case, bind to inaddr_any falls in this
category and need proper handling?
tls-hw-record-offload is TLS record offload to HW, which does tx/rx and record
creation Inline.
enum {
TLS_BASE_TX,
TLS_SW_TX,
TLS_RECORD_HW, /* TLS record processed Inline */
TLS_NUM_CONFIG,
};
-Original Message-
From: Dave Watson [mailto:davejwat...@fb.com]
Sent: Wednesday, January 31, 2018 10:14 PM
To: Atul Gupta
Cc: s...@queasysnail.net; herb...@gondor.apana.org.au;
linux-crypto@vger.kernel.org; ganes...@chelsio.co; net...@vger.kernel.org;
da...@davemloft.net; Boris Pismenny ; Ilya Lesokhin
Subject: Re: [RFC crypto v3 8/9] chtls: Register the ULP
On 01/31/18 04:14 PM, Atul Gupta wrote:
>
>
> On Tuesday 30 January 2018 10:41 PM, Dave Watson wrote:
> > On 01/30/18 06:51 AM, Atul Gupta wrote:
> >
> > > What I was referring is that passing "tls" ulp type in setsockopt
> > > may be insufficient to make the decision when multi HW assist
> > > Inline TLS solution exists.
> > Setting the ULP doesn't choose HW or SW implementation, I think that
> > should be done later when setting up crypto with
> >
> > setsockopt(SOL_TLS, TLS_TX, struct crypto_info).
> setsockpot [mentioned above] is quite late for driver to enable HW
> implementation, we require something as early as tls_init
> [setsockopt(sock, SOL_TCP, TCP_ULP, "tls", sizeof("tls"))], for driver
> to set HW prot and offload connection beside Inline Tx/Rx.
> >
> > Any reason we can't use ethtool to choose HW vs SW implementation,
> > if available on the device?
> Thought about it, the interface index is not available to fetch
> netdev and caps check to set HW prot eg. bind [prot.hash] --> tls_hash to
> program HW.
Perhaps this is the part I don't follow - why do you need to override hash and
check for LISTEN? I briefly looked through the patch named "CPL handler
definition", this looks like it is a full TCP offload?
Yes, this is connection and record layer offload, and the reason I used
different ulp type, need to see what additional info or check can help setup
the required sk prot.
[Crypto v4 00/12] Chelsio Inline TLS
RFC series for Chelsio Inline TLS driver (chtls.ko)
Driver use the ULP infrastructure to register chtls as Inline TLS ULP.
Chtls use TCP Sockets to transmit and receive TLS record. TCP proto_ops
is extended to offload TLS record.
T6 adapter provides the following features:
-TLS record offload, TLS header, encrypt, digest and transmit
-TLS record receive and decrypt
-TLS keys store
-TCP/IP engine
-TLS engine
-GCM crypto engine [support CBC also]
TLS provides security at the transport layer. It uses TCP to provide
reliable end-to-end transport of application data. It relies on TCP
for any retransmission. TLS session comprises of three parts:
a. TCP/IP connection
b. TLS handshake
c. Record layer processing
TLS handshake state machine is executed in host (refer standard
implementation eg. OpenSSL). Setsockopt [SOL_TCP, TCP_ULP] initialize
TCP proto-ops for Chelsio inline tls support. setsockopt(sock, SOL_TCP,
TCP_ULP, "tls", sizeof("tls"));
Tx and Rx Keys are decided during handshake and programmed onto the chip
after CCS is exchanged.
struct tls12_crypto_info_aes_gcm_128 crypto_info
setsockopt(sock, SOL_TLS, TLS_TX, &crypto_info, sizeof(crypto_info))
Finish is the first encrypted/decrypted message tx/rx inline.
On the Tx path TLS engine receive plain text from openssl, insert IV,
fetches the tx key, create cipher text records and generate MAC. TLS
header is added to cipher text and forward to TCP/IP engine for transport
layer processing and transmission on wire.
TX:
Application--openssl--chtls---TLS engine---encrypt/auth---TCP/IP
engine---wire.
On the Rx side, data received is PDU aligned at record
boundaries. TLS processes only the complete record. If rx key is programmed
on CCS receive, data is decrypted and plain text is posted to host.
RX:
Wire--cipher-text--TCP/IP engine [PDU align]---TLS engine---
decrypt/auth---plain-text--chtls--openssl--application
v4: removed chtls ULP type, retained tls ULP
-registered chtls with net tls
-defined struct tls_device to register the Inline drivers
-ethtool interface tls-inline to enable Inline TLS for interface
-prot update to support inline TLS
v3: fixed the kbuild test issues
-made few funtions static
-initialized few variables
v2: fixed the following based on the review comments of Stephan Mueller,
Stefano Brivio and Hannes Frederic
-Added more details in cover letter
-Fixed indentation and formating issues
-Using aes instead of aes-generic
-memset key info after programing the key on chip
-reordered the patch sequence
Atul Gupta (12):
tls: tls_device struct to register TLS drivers
ethtool: feature for Inline TLS in HW
support for inline tls
chtls: structure and macro definiton
cxgb4: Inline TLS FW Interface
cxgb4: LLD driver changes to enable TLS
chcr: Key Macro
chtls: Key program
chtls: CPL handler definition
chtls: Inline crypto request Tx/Rx
chtls: Register the chtls Inline TLS with net tls
Makefile Kconfig
drivers/crypto/chelsio/Kconfig | 11 +
drivers/crypto/chelsio/Makefile|1 +
drivers/crypto/chelsio/chcr_algo.h | 42 +
drivers/crypto/chelsio/chcr_core.h | 55 +-
drivers/crypto/chelsio/chtls/Makefile |4 +
drivers/crypto/chelsio/chtls/chtls.h | 487 +
drivers/crypto/chelsio/chtls/chtls_cm.c| 2045
drivers/crypto/chelsio/chtls/chtls_cm.h| 203 ++
drivers/crypto/chelsio/chtls/chtls_hw.c| 394
drivers/crypto/chelsio/chtls/chtls_io.c| 1867 ++
drivers/crypto/chelsio/chtls/chtls_main.c | 619 ++
drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c | 18 +-
drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c| 32 +-
drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h |7 +
drivers/net/ethernet/chelsio/cxgb4/sge.c | 98 +-
drivers/net/ethernet/chelsio/cxgb4/t4_msg.h| 121 +-
drivers/net/ethernet/chelsio/cxgb4/t4_regs.h |2 +
drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h | 165 +-
include/linux/netdev_features.h|2 +
include/net/tls.h | 21 +
include/uapi/linux/tls.h |1 +
net/core/ethtool.c |1 +
net/ipv4/tcp_minisocks.c |1 +
net/tls/tls_main.c | 113 ++
24 files changed, 6291 insertions(+), 19 deletions(-)
create mode 100644 drivers/crypto/chelsio/chtls/Makefile
create mode 100644 drivers/crypto/chelsio/chtls/chtls.h
create mode 100644 drivers/crypto/chelsio/chtls/chtls_cm.c
create mode 100644 drivers/crypto/chelsio/chtls/chtls_cm.h
create mode 100644 drivers/crypto/chelsio/chtls/chtls_hw.c
create mode 100644 drivers/crypto/chelsio/chtls/chtls_io.c
create mo
[Crypto v4 01/12] tls: tls_device struct to register TLS drivers
added tls_device structure to register Inline TLS
drivers with net/tls
Signed-off-by: Atul Gupta
---
include/net/tls.h | 21 +
1 file changed, 21 insertions(+)
diff --git a/include/net/tls.h b/include/net/tls.h
index 936cfc5..2a9f392 100644
--- a/include/net/tls.h
+++ b/include/net/tls.h
@@ -55,6 +55,25 @@
#define TLS_AAD_SPACE_SIZE 13
+#define TLS_DEVICE_NAME_MAX64
+
+struct tls_device {
+ char name[TLS_DEVICE_NAME_MAX];
+ struct list_head dev_list;
+
+ /* When calling get_netdev, the HW vendor's driver should return the
+* net device of device @device at port @port_num or NULL if such
+* a net device doesn't exist
+*/
+ struct net_device *(*netdev)(struct tls_device *device,
+struct net_device *netdev);
+ int (*feature)(struct tls_device *device);
+ int (*hash)(struct tls_device *device, struct sock *sk);
+ void (*unhash)(struct tls_device *device, struct sock *sk);
+ void (*prot)(struct tls_device *device,
+struct sock *sk);
+};
+
struct tls_sw_context {
struct crypto_aead *aead_send;
@@ -254,5 +273,7 @@ static inline struct tls_offload_context *tls_offload_ctx(
int tls_proccess_cmsg(struct sock *sk, struct msghdr *msg,
unsigned char *record_type);
+void tls_register_device(struct tls_device *device);
+void tls_unregister_device(struct tls_device *device);
#endif /* _TLS_OFFLOAD_H */
--
1.8.3.1
[Crypto v4 02/12] ethtool: feature for Inline TLS in HW
Signed-off-by: Atul Gupta
---
include/linux/netdev_features.h | 2 ++
net/core/ethtool.c | 1 +
2 files changed, 3 insertions(+)
diff --git a/include/linux/netdev_features.h b/include/linux/netdev_features.h
index b1b0ca7..e1a33b7 100644
--- a/include/linux/netdev_features.h
+++ b/include/linux/netdev_features.h
@@ -77,6 +77,7 @@ enum {
NETIF_F_HW_ESP_BIT, /* Hardware ESP transformation offload
*/
NETIF_F_HW_ESP_TX_CSUM_BIT, /* ESP with TX checksum offload */
NETIF_F_RX_UDP_TUNNEL_PORT_BIT, /* Offload of RX port for UDP tunnels */
+ NETIF_F_HW_TLS_INLINE_BIT, /* Offload TLS record */
/*
* Add your fresh new feature above and remember to update
@@ -142,6 +143,7 @@ enum {
#define NETIF_F_HW_ESP __NETIF_F(HW_ESP)
#define NETIF_F_HW_ESP_TX_CSUM __NETIF_F(HW_ESP_TX_CSUM)
#defineNETIF_F_RX_UDP_TUNNEL_PORT __NETIF_F(RX_UDP_TUNNEL_PORT)
+#define NETIF_F_HW_TLS_INLINE __NETIF_F(HW_TLS_INLINE)
#define for_each_netdev_feature(mask_addr, bit)\
for_each_set_bit(bit, (unsigned long *)mask_addr, NETDEV_FEATURE_COUNT)
diff --git a/net/core/ethtool.c b/net/core/ethtool.c
index f8fcf45..cac1c77 100644
--- a/net/core/ethtool.c
+++ b/net/core/ethtool.c
@@ -106,6 +106,7 @@ int ethtool_op_get_ts_info(struct net_device *dev, struct
ethtool_ts_info *info)
[NETIF_F_HW_ESP_BIT] = "esp-hw-offload",
[NETIF_F_HW_ESP_TX_CSUM_BIT] = "esp-tx-csum-hw-offload",
[NETIF_F_RX_UDP_TUNNEL_PORT_BIT] = "rx-udp_tunnel-port-offload",
+ [NETIF_F_HW_TLS_INLINE_BIT] ="tls-inline",
};
static const char
--
1.8.3.1
[Crypto v4 03/12] support for inline tls
Signed-off-by: Atul Gupta
---
net/tls/tls_main.c | 113 +
1 file changed, 113 insertions(+)
diff --git a/net/tls/tls_main.c b/net/tls/tls_main.c
index e07ee3a..10a6d5d 100644
--- a/net/tls/tls_main.c
+++ b/net/tls/tls_main.c
@@ -38,6 +38,7 @@
#include
#include
#include
+#include
#include
@@ -48,9 +49,12 @@
enum {
TLS_BASE_TX,
TLS_SW_TX,
+ TLS_FULL_HW, /* TLS record processed Inline */
TLS_NUM_CONFIG,
};
+static LIST_HEAD(device_list);
+static DEFINE_MUTEX(device_mutex);
static struct proto tls_prots[TLS_NUM_CONFIG];
static inline void update_sk_prot(struct sock *sk, struct tls_context *ctx)
@@ -448,6 +452,92 @@ static int tls_setsockopt(struct sock *sk, int level, int
optname,
return do_tls_setsockopt(sk, optname, optval, optlen);
}
+static struct net_device *find_netdev(struct sock *sk)
+{
+ struct net_device *netdev = NULL;
+
+ netdev = __ip_dev_find(&init_net, inet_sk(sk)->inet_rcv_saddr, false);
+ return netdev;
+}
+
+static int get_tls_prot(struct sock *sk)
+{
+ struct tls_context *ctx = tls_get_ctx(sk);
+ struct net_device *netdev;
+ struct tls_device *dev;
+
+ /* Device bound to specific IP */
+ if (inet_sk(sk)->inet_rcv_saddr) {
+ netdev = find_netdev(sk);
+ if (!netdev)
+ goto out;
+
+ /* Device supports Inline record processing */
+ if (!(netdev->features & NETIF_F_HW_TLS_INLINE))
+ goto out;
+
+ mutex_lock(&device_mutex);
+ list_for_each_entry(dev, &device_list, dev_list) {
+ if (dev->netdev && dev->netdev(dev, netdev))
+ break;
+ }
+ mutex_unlock(&device_mutex);
+
+ ctx->tx_conf = TLS_FULL_HW;
+ if (dev->prot)
+ dev->prot(dev, sk);
+ } else { /* src address not known or INADDR_ANY */
+ mutex_lock(&device_mutex);
+ list_for_each_entry(dev, &device_list, dev_list) {
+ if (dev->feature && dev->feature(dev)) {
+ ctx->tx_conf = TLS_FULL_HW;
+ break;
+ }
+ }
+ mutex_unlock(&device_mutex);
+ update_sk_prot(sk, ctx);
+ }
+out:
+ return ctx->tx_conf;
+}
+
+static int tls_hw_prot(struct sock *sk)
+{
+ /* search registered tls device for netdev */
+ return get_tls_prot(sk);
+}
+
+static void tls_hw_unhash(struct sock *sk)
+{
+ struct tls_device *dev;
+
+ mutex_lock(&device_mutex);
+ list_for_each_entry(dev, &device_list, dev_list) {
+ if (dev->unhash)
+ dev->unhash(dev, sk);
+ }
+ mutex_unlock(&device_mutex);
+ tcp_prot.unhash(sk);
+}
+
+static int tls_hw_hash(struct sock *sk)
+{
+ struct tls_device *dev;
+ int err;
+
+ err = tcp_prot.hash(sk);
+ mutex_lock(&device_mutex);
+ list_for_each_entry(dev, &device_list, dev_list) {
+ if (dev->hash)
+ err |= dev->hash(dev, sk);
+ }
+ mutex_unlock(&device_mutex);
+
+ if (err)
+ tls_hw_unhash(sk);
+ return err;
+}
+
static int tls_init(struct sock *sk)
{
struct inet_connection_sock *icsk = inet_csk(sk);
@@ -466,6 +556,9 @@ static int tls_init(struct sock *sk)
ctx->sk_proto_close = sk->sk_prot->close;
ctx->tx_conf = TLS_BASE_TX;
+ if (tls_hw_prot(sk) == TLS_FULL_HW)
+ goto out;
+
update_sk_prot(sk, ctx);
out:
return rc;
@@ -487,7 +580,27 @@ static void build_protos(struct proto *prot, struct proto
*base)
prot[TLS_SW_TX] = prot[TLS_BASE_TX];
prot[TLS_SW_TX].sendmsg = tls_sw_sendmsg;
prot[TLS_SW_TX].sendpage= tls_sw_sendpage;
+
+ prot[TLS_FULL_HW] = prot[TLS_BASE_TX];
+ prot[TLS_FULL_HW].hash = tls_hw_hash;
+ prot[TLS_FULL_HW].unhash= tls_hw_unhash;
+}
+
+void tls_register_device(struct tls_device *device)
+{
+ mutex_lock(&device_mutex);
+ list_add_tail(&device->dev_list, &device_list);
+ mutex_unlock(&device_mutex);
+}
+EXPORT_SYMBOL(tls_register_device);
+
+void tls_unregister_device(struct tls_device *device)
+{
+ mutex_lock(&device_mutex);
+ list_del(&device->dev_list);
+ mutex_unlock(&device_mutex);
}
+EXPORT_SYMBOL(tls_unregister_device);
static int __init tls_register(void)
{
--
1.8.3.1
[Crypto v4 04/12] chtls: structure and macro definiton
Inline TLS state, connection management. Supporting macros definition.
Signed-off-by: Atul Gupta
---
drivers/crypto/chelsio/chtls/chtls.h| 487
drivers/crypto/chelsio/chtls/chtls_cm.h | 203 +
2 files changed, 690 insertions(+)
create mode 100644 drivers/crypto/chelsio/chtls/chtls.h
create mode 100644 drivers/crypto/chelsio/chtls/chtls_cm.h
diff --git a/drivers/crypto/chelsio/chtls/chtls.h
b/drivers/crypto/chelsio/chtls/chtls.h
new file mode 100644
index 000..c7b8d59
--- /dev/null
+++ b/drivers/crypto/chelsio/chtls/chtls.h
@@ -0,0 +1,487 @@
+/*
+ * Copyright (c) 2016 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef __CHTLS_H__
+#define __CHTLS_H__
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "t4fw_api.h"
+#include "t4_msg.h"
+#include "cxgb4.h"
+#include "cxgb4_uld.h"
+#include "l2t.h"
+#include "chcr_algo.h"
+#include "chcr_core.h"
+#include "chcr_crypto.h"
+
+#define CIPHER_BLOCK_SIZE 16
+#define MAX_IVS_PAGE256
+#define TLS_KEY_CONTEXT_SZ 64
+#define TLS_HEADER_LENGTH 5
+#define SCMD_CIPH_MODE_AES_GCM 2
+#define GCM_TAG_SIZE16
+#define AEAD_EXPLICIT_DATA_SIZE 8
+/* Any MFS size should work and come from openssl */
+#define TLS_MFS16384
+
+#define SOCK_INLINE (31)
+#define RSS_HDR sizeof(struct rss_header)
+
+enum {
+ CHTLS_KEY_CONTEXT_DSGL,
+ CHTLS_KEY_CONTEXT_IMM,
+ CHTLS_KEY_CONTEXT_DDR,
+};
+
+enum {
+ CHTLS_LISTEN_START,
+ CHTLS_LISTEN_STOP,
+};
+
+/* Flags for return value of CPL message handlers */
+enum {
+ CPL_RET_BUF_DONE = 1, /* buffer processing done */
+ CPL_RET_BAD_MSG = 2,/* bad CPL message */
+ CPL_RET_UNKNOWN_TID = 4 /* unexpected unknown TID */
+};
+
+#define TLS_RCV_ST_READ_HEADER 0xF0
+#define TLS_RCV_ST_READ_BODY0xF1
+#define TLS_RCV_ST_READ_DONE0xF2
+#define TLS_RCV_ST_READ_NB 0xF3
+
+#define RSPQ_HASH_BITS 5
+#define LISTEN_INFO_HASH_SIZE 32
+struct listen_info {
+ struct listen_info *next; /* Link to next entry */
+ struct sock *sk; /* The listening socket */
+ unsigned int stid; /* The server TID */
+};
+
+enum {
+ T4_LISTEN_START_PENDING,
+ T4_LISTEN_STARTED
+};
+
+enum csk_flags {
+ CSK_CALLBACKS_CHKD, /* socket callbacks have been sanitized */
+ CSK_ABORT_REQ_RCVD, /* received one ABORT_REQ_RSS message */
+ CSK_TX_MORE_DATA, /* sending ULP data; don't set SHOVE bit */
+ CSK_TX_WAIT_IDLE, /* suspend Tx until in-flight data is ACKed */
+ CSK_ABORT_SHUTDOWN, /* shouldn't send more abort requests */
+ CSK_ABORT_RPL_PENDING, /* expecting an abort reply */
+ CSK_CLOSE_CON_REQUESTED,/* we've sent a close_conn_req */
+ CSK_TX_DATA_SENT, /* sent a TX_DATA WR on this connection */
+ CSK_TX_FAILOVER,/* Tx traffic failing over */
+ CSK_UPDATE_RCV_WND, /* Need to update rcv window */
+ CSK_RST_ABORTED,/* outgoing RST was aborted */
+ CSK_TLS_HANDSHK,/* TLS Handshake */
+};
+
+struct listen_ctx {
+ struct sock *lsk;
+ struct chtls_dev *cdev;
+ u32 state;
+};
+
+struct key_map {
+ unsigned long *addr;
+ unsigned int start;
+ unsigned int available;
+ unsigned int size;
+ spinlock_t lock; /* lock for key id request from map */
+} __packed;
+
+struct tls_scmd {
+ __be32 seqno_numivs;
+ __be32 ivgen_hdrlen;
+};
+
+struct chtls_dev {
+ struct tls_device tlsdev;
+ struct list_head list;
+ struct cxgb4_lld_info *lldi;
+ struct pci_dev *pdev;
+ struct listen_info *listen_hash_tab[LISTEN_INFO_HASH_SIZE];
+ spinlock_t listen_lock; /* lock for listen list */
+ struct net_device **ports;
+ struct tid_info *tids;
+ unsigned int pfvf;
+ const unsigned short *mtus;
+
+ spinlock_t aidr_lock cacheline_aligned_in_smp;
+ struct idr aidr; /* ATID id space */
+ struct idr hwtid_idr;
+ struct idr stid_idr;
+
+ spinlock_t idr_lock cacheline_aligned_in_smp;
+
+ struct net_device *egr_dev[NCHAN * 2];
+ struct sk_buff *rspq_skb_cache[1 << RSPQ_HASH_BITS];
+ struct sk_buff *askb;
+
+ struct sk_buff_head deferq;
+ struct work_struct deferq_task;
+
+ struct list_head list_node;
+ struct list_head rcu_node;
+ struct list_head na_node;
+
[Crypto v4 05/12] cxgb4: Inline TLS FW Interface
Key area size in hw-config file. CPL struct for TLS request
and response. Work request for Inline TLS.
Signed-off-by: Atul Gupta
---
drivers/net/ethernet/chelsio/cxgb4/t4_msg.h | 121 ++-
drivers/net/ethernet/chelsio/cxgb4/t4_regs.h | 2 +
drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h | 165 +-
3 files changed, 283 insertions(+), 5 deletions(-)
diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h
b/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h
index 7e12f24..9a56e0d 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h
@@ -81,6 +81,7 @@ enum {
CPL_RX_ISCSI_CMP = 0x45,
CPL_TRACE_PKT_T5 = 0x48,
CPL_RX_ISCSI_DDP = 0x49,
+ CPL_RX_TLS_CMP= 0x4E,
CPL_RDMA_READ_REQ = 0x60,
@@ -88,6 +89,7 @@ enum {
CPL_ACT_OPEN_REQ6 = 0x83,
CPL_TX_TLS_PDU =0x88,
+ CPL_TX_TLS_SFO= 0x89,
CPL_TX_SEC_PDU= 0x8A,
CPL_TX_TLS_ACK= 0x8B,
@@ -97,6 +99,7 @@ enum {
CPL_RX_MPS_PKT= 0xAF,
CPL_TRACE_PKT = 0xB0,
+ CPL_TLS_DATA = 0xB1,
CPL_ISCSI_DATA= 0xB2,
CPL_FW4_MSG = 0xC0,
@@ -150,6 +153,7 @@ enum {
ULP_MODE_RDMA = 4,
ULP_MODE_TCPDDP= 5,
ULP_MODE_FCOE = 6,
+ ULP_MODE_TLS = 8,
};
enum {
@@ -1414,6 +1418,14 @@ struct cpl_tx_data {
#define TX_FORCE_S 13
#define TX_FORCE_V(x) ((x) << TX_FORCE_S)
+#define TX_SHOVE_S14
+#define TX_SHOVE_V(x) ((x) << TX_SHOVE_S)
+
+#define TX_ULP_MODE_S10
+#define TX_ULP_MODE_M0x7
+#define TX_ULP_MODE_V(x) ((x) << TX_ULP_MODE_S)
+#define TX_ULP_MODE_G(x) (((x) >> TX_ULP_MODE_S) & TX_ULP_MODE_M)
+
#define T6_TX_FORCE_S 20
#define T6_TX_FORCE_V(x) ((x) << T6_TX_FORCE_S)
#define T6_TX_FORCE_F T6_TX_FORCE_V(1U)
@@ -1428,12 +1440,21 @@ enum {
ULP_TX_SC_NOOP = 0x80,
ULP_TX_SC_IMM = 0x81,
ULP_TX_SC_DSGL = 0x82,
- ULP_TX_SC_ISGL = 0x83
+ ULP_TX_SC_ISGL = 0x83,
+ ULP_TX_SC_MEMRD = 0x86
};
#define ULPTX_CMD_S24
#define ULPTX_CMD_V(x) ((x) << ULPTX_CMD_S)
+#define ULPTX_LEN16_S0
+#define ULPTX_LEN16_M0xFF
+#define ULPTX_LEN16_V(x) ((x) << ULPTX_LEN16_S)
+
+#define ULP_TX_SC_MORE_S 23
+#define ULP_TX_SC_MORE_V(x) ((x) << ULP_TX_SC_MORE_S)
+#define ULP_TX_SC_MORE_F ULP_TX_SC_MORE_V(1U)
+
struct ulptx_sge_pair {
__be32 len[2];
__be64 addr[2];
@@ -1948,4 +1969,102 @@ enum {
X_CPL_RX_MPS_PKT_TYPE_QFC = 1 << 2,
X_CPL_RX_MPS_PKT_TYPE_PTP = 1 << 3
};
+
+struct cpl_tx_tls_sfo {
+ __be32 op_to_seg_len;
+ __be32 pld_len;
+ __be32 type_protover;
+ __be32 r1_lo;
+ __be32 seqno_numivs;
+ __be32 ivgen_hdrlen;
+ __be64 scmd1;
+};
+
+/* cpl_tx_tls_sfo macros */
+#define CPL_TX_TLS_SFO_OPCODE_S 24
+#define CPL_TX_TLS_SFO_OPCODE_V(x) ((x) << CPL_TX_TLS_SFO_OPCODE_S)
+
+#define CPL_TX_TLS_SFO_DATA_TYPE_S 20
+#define CPL_TX_TLS_SFO_DATA_TYPE_V(x) ((x) << CPL_TX_TLS_SFO_DATA_TYPE_S)
+
+#define CPL_TX_TLS_SFO_CPL_LEN_S16
+#define CPL_TX_TLS_SFO_CPL_LEN_V(x) ((x) << CPL_TX_TLS_SFO_CPL_LEN_S)
+
+#define CPL_TX_TLS_SFO_SEG_LEN_S0
+#define CPL_TX_TLS_SFO_SEG_LEN_M0x
+#define CPL_TX_TLS_SFO_SEG_LEN_V(x) ((x) << CPL_TX_TLS_SFO_SEG_LEN_S)
+#define CPL_TX_TLS_SFO_SEG_LEN_G(x) \
+ (((x) >> CPL_TX_TLS_SFO_SEG_LEN_S) & CPL_TX_TLS_SFO_SEG_LEN_M)
+
+#define CPL_TX_TLS_SFO_TYPE_S 24
+#define CPL_TX_TLS_SFO_TYPE_M 0xff
+#define CPL_TX_TLS_SFO_TYPE_V(x)((x) << CPL_TX_TLS_SFO_TYPE_S)
+#define CPL_TX_TLS_SFO_TYPE_G(x)\
+ (((x) >> CPL_TX_TLS_SFO_TYPE_S) & CPL_TX_TLS_SFO_TYPE_M)
+
+#define CPL_TX_TLS_SFO_PROTOVER_S 8
+#define CPL_TX_TLS_SFO_PROTOVER_M 0x
+#define CPL_TX_TLS_SFO_PROTOVER_V(x)((x) << CPL_TX_TLS_SFO_PROTOVER_S)
+#define CPL_TX_TLS_SFO_PROTOVER_G(x)\
+ (((x) >> CPL_TX_TLS_SFO_PROTOVER_S) & CPL_TX_TLS_SFO_PROTOVER_M)
+
+struct cpl_tls_data {
+ struct rss_header rsshdr;
+ union opcode_tid ot;
+ __be32 length_pkd;
+ __be32 seq;
+ __be32 r1;
+};
+
+#define CPL_TLS_DATA_OPCODE_S 24
+#define CPL_TLS_DATA_OPCODE_M 0xff
+#define CPL_TLS_DATA_OPCODE_V(x)((x) << CPL_TLS_DATA_OPCODE_S)
+#define CPL_TLS_DATA_OPCODE_G(x)\
+ (((x) >> CPL_TLS_DATA_OPCODE_S) & CPL_TLS_DATA_OPCODE_M)
+
+#define CPL_TLS_DATA_TID_S 0
+#define CPL_TLS_DATA_TID_M 0xff
+#define CPL_TLS_DATA_TID_V(x) ((x) << CPL_TLS_DATA_TID_S)
+#define CPL_TLS_DATA_TID_G(x) \
+ (((x) >> CPL_TLS_DATA_TID_S) &
[Crypto v4 07/12] chcr: Key Macro
Define macro for TLS Key context
Signed-off-by: Atul Gupta
---
drivers/crypto/chelsio/chcr_algo.h | 42 +
drivers/crypto/chelsio/chcr_core.h | 55 +-
2 files changed, 96 insertions(+), 1 deletion(-)
diff --git a/drivers/crypto/chelsio/chcr_algo.h
b/drivers/crypto/chelsio/chcr_algo.h
index d1673a5..f263cd4 100644
--- a/drivers/crypto/chelsio/chcr_algo.h
+++ b/drivers/crypto/chelsio/chcr_algo.h
@@ -86,6 +86,39 @@
KEY_CONTEXT_OPAD_PRESENT_M)
#define KEY_CONTEXT_OPAD_PRESENT_F KEY_CONTEXT_OPAD_PRESENT_V(1U)
+#define TLS_KEYCTX_RXFLIT_CNT_S 24
+#define TLS_KEYCTX_RXFLIT_CNT_V(x) ((x) << TLS_KEYCTX_RXFLIT_CNT_S)
+
+#define TLS_KEYCTX_RXPROT_VER_S 20
+#define TLS_KEYCTX_RXPROT_VER_M 0xf
+#define TLS_KEYCTX_RXPROT_VER_V(x) ((x) << TLS_KEYCTX_RXPROT_VER_S)
+
+#define TLS_KEYCTX_RXCIPH_MODE_S 16
+#define TLS_KEYCTX_RXCIPH_MODE_M 0xf
+#define TLS_KEYCTX_RXCIPH_MODE_V(x) ((x) << TLS_KEYCTX_RXCIPH_MODE_S)
+
+#define TLS_KEYCTX_RXAUTH_MODE_S 12
+#define TLS_KEYCTX_RXAUTH_MODE_M 0xf
+#define TLS_KEYCTX_RXAUTH_MODE_V(x) ((x) << TLS_KEYCTX_RXAUTH_MODE_S)
+
+#define TLS_KEYCTX_RXCIAU_CTRL_S 11
+#define TLS_KEYCTX_RXCIAU_CTRL_V(x) ((x) << TLS_KEYCTX_RXCIAU_CTRL_S)
+
+#define TLS_KEYCTX_RX_SEQCTR_S 9
+#define TLS_KEYCTX_RX_SEQCTR_M 0x3
+#define TLS_KEYCTX_RX_SEQCTR_V(x) ((x) << TLS_KEYCTX_RX_SEQCTR_S)
+
+#define TLS_KEYCTX_RX_VALID_S 8
+#define TLS_KEYCTX_RX_VALID_V(x) ((x) << TLS_KEYCTX_RX_VALID_S)
+
+#define TLS_KEYCTX_RXCK_SIZE_S 3
+#define TLS_KEYCTX_RXCK_SIZE_M 0x7
+#define TLS_KEYCTX_RXCK_SIZE_V(x) ((x) << TLS_KEYCTX_RXCK_SIZE_S)
+
+#define TLS_KEYCTX_RXMK_SIZE_S 0
+#define TLS_KEYCTX_RXMK_SIZE_M 0x7
+#define TLS_KEYCTX_RXMK_SIZE_V(x) ((x) << TLS_KEYCTX_RXMK_SIZE_S)
+
#define CHCR_HASH_MAX_DIGEST_SIZE 64
#define CHCR_MAX_SHA_DIGEST_SIZE 64
@@ -176,6 +209,15 @@
KEY_CONTEXT_SALT_PRESENT_V(1) | \
KEY_CONTEXT_CTX_LEN_V((ctx_len)))
+#define FILL_KEY_CRX_HDR(ck_size, mk_size, d_ck, opad, ctx_len) \
+ htonl(TLS_KEYCTX_RXMK_SIZE_V(mk_size) | \
+ TLS_KEYCTX_RXCK_SIZE_V(ck_size) | \
+ TLS_KEYCTX_RX_VALID_V(1) | \
+ TLS_KEYCTX_RX_SEQCTR_V(3) | \
+ TLS_KEYCTX_RXAUTH_MODE_V(4) | \
+ TLS_KEYCTX_RXCIPH_MODE_V(2) | \
+ TLS_KEYCTX_RXFLIT_CNT_V((ctx_len)))
+
#define FILL_WR_OP_CCTX_SIZE \
htonl( \
FW_CRYPTO_LOOKASIDE_WR_OPCODE_V( \
diff --git a/drivers/crypto/chelsio/chcr_core.h
b/drivers/crypto/chelsio/chcr_core.h
index 3c29ee0..77056a9 100644
--- a/drivers/crypto/chelsio/chcr_core.h
+++ b/drivers/crypto/chelsio/chcr_core.h
@@ -65,10 +65,58 @@
struct _key_ctx {
__be32 ctx_hdr;
u8 salt[MAX_SALT];
- __be64 reserverd;
+ __be64 iv_to_auth;
unsigned char key[0];
};
+#define KEYCTX_TX_WR_IV_S 55
+#define KEYCTX_TX_WR_IV_M 0x1ffULL
+#define KEYCTX_TX_WR_IV_V(x) ((x) << KEYCTX_TX_WR_IV_S)
+#define KEYCTX_TX_WR_IV_G(x) \
+ (((x) >> KEYCTX_TX_WR_IV_S) & KEYCTX_TX_WR_IV_M)
+
+#define KEYCTX_TX_WR_AAD_S 47
+#define KEYCTX_TX_WR_AAD_M 0xffULL
+#define KEYCTX_TX_WR_AAD_V(x) ((x) << KEYCTX_TX_WR_AAD_S)
+#define KEYCTX_TX_WR_AAD_G(x) (((x) >> KEYCTX_TX_WR_AAD_S) & \
+ KEYCTX_TX_WR_AAD_M)
+
+#define KEYCTX_TX_WR_AADST_S 39
+#define KEYCTX_TX_WR_AADST_M 0xffULL
+#define KEYCTX_TX_WR_AADST_V(x) ((x) << KEYCTX_TX_WR_AADST_S)
+#define KEYCTX_TX_WR_AADST_G(x) \
+ (((x) >> KEYCTX_TX_WR_AADST_S) & KEYCTX_TX_WR_AADST_M)
+
+#define KEYCTX_TX_WR_CIPHER_S 30
+#define KEYCTX_TX_WR_CIPHER_M 0x1ffULL
+#define KEYCTX_TX_WR_CIPHER_V(x) ((x) << KEYCTX_TX_WR_CIPHER_S)
+#define KEYCTX_TX_WR_CIPHER_G(x) \
+ (((x) >> KEYCTX_TX_WR_CIPHER_S) & KEYCTX_TX_WR_CIPHER_M)
+
+#define KEYCTX_TX_WR_CIPHERST_S 23
+#define KEYCTX_TX_WR_CIPHERST_M 0x7f
+#define KEYCTX_TX_WR_CIPHERST_V(x) ((x) << KEYCTX_TX_WR_CIPHERST_S)
+#define KEYCTX_TX_WR_CIPHERST_G(x) \
+ (((x) >> KEYCTX_TX_WR_CIPHERST_S) & KEYCTX_TX_WR_CIPHERST_M)
+
+#define KEYCTX_TX_WR_AUTH_S 14
+#define KEYCTX_TX_WR_AUTH_M 0x1ff
+#define KEYCTX_TX_WR_AUTH_V(x) ((x) << KEYCTX_TX_WR_AUTH_S)
+#define KEYCTX_TX_WR_AUTH_G(x) \
+ (((x) >> KEYCTX_TX_WR_AUTH_S) & KEYCTX_TX_WR_AUTH_M)
+
+#define KEYCTX_TX_WR_AUTHST_S 7
+#define KEYCTX_TX_WR_AUTHST_M 0x7f
+#define KEYCTX_TX_WR_AUTHST_V(x) ((x) << KEYCTX_TX_WR_AUTHST_S)
+#define KEYCTX_TX_WR_AUTHST_G(x) \
+ (((x) >> KEYCTX_TX_WR_AUTHST_S) & KEYCTX_TX_WR_AUTHST_M)
+
+#define KEYCTX_TX_WR_AUTHIN_S 0
+#define KEYCTX_TX_WR_AUTHIN_M 0x7f
+#define KEYCTX_TX_WR_AUTHIN_V(x) ((x) << KEYCTX_TX_WR_AUTHIN_S)
+#define KEYCTX_TX_WR_AUTHIN_G(x) \
+ (((x) >> KEYCTX_TX_WR_AUTHIN_S) & KEYCTX_TX_WR_AUTH
[Crypto v4 08/12] chtls: Key program
Program the tx and rx key on chip.
Signed-off-by: Atul Gupta
---
drivers/crypto/chelsio/chtls/chtls_hw.c | 394
1 file changed, 394 insertions(+)
create mode 100644 drivers/crypto/chelsio/chtls/chtls_hw.c
diff --git a/drivers/crypto/chelsio/chtls/chtls_hw.c
b/drivers/crypto/chelsio/chtls/chtls_hw.c
new file mode 100644
index 000..c3e17159
--- /dev/null
+++ b/drivers/crypto/chelsio/chtls/chtls_hw.c
@@ -0,0 +1,394 @@
+/*
+ * Copyright (c) 2017 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Written by: Atul Gupta (atul.gu...@chelsio.com)
+ */
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "chtls.h"
+#include "chtls_cm.h"
+
+static void __set_tcb_field_direct(struct chtls_sock *csk,
+ struct cpl_set_tcb_field *req, u16 word,
+ u64 mask, u64 val, u8 cookie, int no_reply)
+{
+ struct ulptx_idata *sc;
+
+ INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, csk->tid);
+ req->wr.wr_mid |= htonl(FW_WR_FLOWID_V(csk->tid));
+ req->reply_ctrl = htons(NO_REPLY_V(no_reply) |
+ QUEUENO_V(csk->rss_qid));
+ req->word_cookie = htons(TCB_WORD_V(word) | TCB_COOKIE_V(cookie));
+ req->mask = cpu_to_be64(mask);
+ req->val = cpu_to_be64(val);
+ sc = (struct ulptx_idata *)(req + 1);
+ sc->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP));
+ sc->len = htonl(0);
+}
+
+static void __set_tcb_field(struct sock *sk, struct sk_buff *skb, u16 word,
+ u64 mask, u64 val, u8 cookie, int no_reply)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+ struct cpl_set_tcb_field *req;
+ struct ulptx_idata *sc;
+ unsigned int wrlen = roundup(sizeof(*req) + sizeof(*sc), 16);
+
+ req = (struct cpl_set_tcb_field *)__skb_put(skb, wrlen);
+ __set_tcb_field_direct(csk, req, word, mask, val, cookie, no_reply);
+ set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id);
+}
+
+static int chtls_set_tcb_field(struct sock *sk, u16 word, u64 mask, u64 val)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+ struct sk_buff *skb;
+ struct cpl_set_tcb_field *req;
+ struct ulptx_idata *sc;
+ unsigned int wrlen = roundup(sizeof(*req) + sizeof(*sc), 16);
+ unsigned int credits_needed = DIV_ROUND_UP(wrlen, 16);
+
+ skb = alloc_skb(wrlen, GFP_ATOMIC);
+ if (!skb)
+ return -ENOMEM;
+
+ __set_tcb_field(sk, skb, word, mask, val, 0, 1);
+ set_queue(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA, sk);
+ csk->wr_credits -= credits_needed;
+ csk->wr_unacked += credits_needed;
+ enqueue_wr(csk, skb);
+ cxgb4_ofld_send(csk->egress_dev, skb);
+ return 0;
+}
+
+/*
+ * Set one of the t_flags bits in the TCB.
+ */
+int chtls_set_tcb_tflag(struct sock *sk, unsigned int bit_pos, int val)
+{
+ return chtls_set_tcb_field(sk, 1, 1ULL << bit_pos,
+ val << bit_pos);
+}
+
+static int chtls_set_tcb_keyid(struct sock *sk, int keyid)
+{
+ return chtls_set_tcb_field(sk, 31, 0xULL, keyid);
+}
+
+static int chtls_set_tcb_seqno(struct sock *sk)
+{
+ return chtls_set_tcb_field(sk, 28, ~0ULL, 0);
+}
+
+static int chtls_set_tcb_quiesce(struct sock *sk, int val)
+{
+ return chtls_set_tcb_field(sk, 1, (1ULL << TF_RX_QUIESCE_S),
+ TF_RX_QUIESCE_V(val));
+}
+
+static void *chtls_alloc_mem(unsigned long size)
+{
+ void *p = kmalloc(size, GFP_KERNEL);
+
+ if (!p)
+ p = vmalloc(size);
+ if (p)
+ memset(p, 0, size);
+ return p;
+}
+
+static void chtls_free_mem(void *addr)
+{
+ unsigned long p = (unsigned long)addr;
+
+ if (p >= VMALLOC_START && p < VMALLOC_END)
+ vfree(addr);
+ else
+ kfree(addr);
+}
+
+/* TLS Key bitmap processing */
+int chtls_init_kmap(struct chtls_dev *cdev, struct cxgb4_lld_info *lldi)
+{
+ unsigned int num_key_ctx, bsize;
+
+ num_key_ctx = (lldi->vr->key.size / TLS_KEY_CONTEXT_SZ);
+ bsize = BITS_TO_LONGS(num_key_ctx);
+
+ cdev->kmap.size = num_key_ctx;
+ cdev->kmap.available = bsize;
+ cdev->kmap.addr = chtls_alloc_mem(sizeof(*cdev->kmap.addr) *
+ bsize);
+ if (!cdev->kmap.addr)
+ return -1;
+
+ cdev->kmap.start = lldi->vr->key.start;
+ spin_lock_init(&cdev->kmap.lock);
+ return 0;
+}
+
+void chtls_free_kmap(st
[Crypto v4 06/12] cxgb4: LLD driver changes to enable TLS
Read FW capability. Read key area size. Dump the TLS record count.
Signed-off-by: Atul Gupta
---
drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c | 18 +++-
drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c| 32 +--
drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h | 7 ++
drivers/net/ethernet/chelsio/cxgb4/sge.c | 98 +-
4 files changed, 142 insertions(+), 13 deletions(-)
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
index cf47183..cfc9210 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
@@ -2826,8 +2826,8 @@ static int meminfo_show(struct seq_file *seq, void *v)
"Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
"TDDP region:", "TPT region:", "STAG region:", "RQ region:",
"RQUDP region:", "PBL region:", "TXPBL region:",
- "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
- "On-chip queues:"
+ "TLSKey region:", "DBVFIFO region:", "ULPRX state:",
+ "ULPTX state:", "On-chip queues:"
};
int i, n;
@@ -2943,6 +2943,12 @@ static int meminfo_show(struct seq_file *seq, void *v)
ulp_region(RX_RQUDP);
ulp_region(RX_PBL);
ulp_region(TX_PBL);
+ if (adap->params.crypto & FW_CAPS_CONFIG_TLS_INLINE) {
+ ulp_region(RX_TLS_KEY);
+ } else {
+ md->base = 0;
+ md->idx = ARRAY_SIZE(region);
+ }
#undef ulp_region
md->base = 0;
md->idx = ARRAY_SIZE(region);
@@ -3098,6 +3104,14 @@ static int chcr_show(struct seq_file *seq, void *v)
atomic_read(&adap->chcr_stats.fallback));
seq_printf(seq, "IPSec PDU: %10u\n",
atomic_read(&adap->chcr_stats.ipsec_cnt));
+
+ seq_puts(seq, "\nChelsio Inline TLS Stats\n");
+ seq_printf(seq, "TLS PDU Tx: %u\n",
+ atomic_read(&adap->chcr_stats.tls_pdu_tx));
+ seq_printf(seq, "TLS PDU Rx: %u\n",
+ atomic_read(&adap->chcr_stats.tls_pdu_rx));
+ seq_printf(seq, "TLS Keys (DDR) Count: %u\n",
+ atomic_read(&adap->chcr_stats.tls_key));
return 0;
}
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
index 05a4abf..60eb18b 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
@@ -4086,18 +4086,32 @@ static int adap_init0(struct adapter *adap)
adap->num_ofld_uld += 2;
}
if (caps_cmd.cryptocaps) {
- /* Should query params here...TODO */
- params[0] = FW_PARAM_PFVF(NCRYPTO_LOOKASIDE);
- ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2,
- params, val);
- if (ret < 0) {
- if (ret != -EINVAL)
+ if (ntohs(caps_cmd.cryptocaps) &
+ FW_CAPS_CONFIG_CRYPTO_LOOKASIDE) {
+ params[0] = FW_PARAM_PFVF(NCRYPTO_LOOKASIDE);
+ ret = t4_query_params(adap, adap->mbox, adap->pf, 0,
+ 2, params, val);
+ if (ret < 0) {
+ if (ret != -EINVAL)
+ goto bye;
+ } else {
+ adap->vres.ncrypto_fc = val[0];
+ }
+ adap->num_ofld_uld += 1;
+ }
+ if (ntohs(caps_cmd.cryptocaps) &
+ FW_CAPS_CONFIG_TLS_INLINE) {
+ params[0] = FW_PARAM_PFVF(TLS_START);
+ params[1] = FW_PARAM_PFVF(TLS_END);
+ ret = t4_query_params(adap, adap->mbox, adap->pf, 0,
+ 2, params, val);
+ if (ret < 0)
goto bye;
- } else {
- adap->vres.ncrypto_fc = val[0];
+ adap->vres.key.start = val[0];
+ adap->vres.key.size = val[1] - val[0] + 1;
+ adap->num_uld += 1;
}
adap->params.crypto = ntohs(caps_cmd.cryptocaps);
- adap->num_uld += 1;
}
#undef FW_PARAM_PFVF
#undef FW_PARAM_DEV
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h
b/drivers/
[Crypto v4 10/12] chtls: Inline crypto request Tx/Rx
TLS handler for record transmit and receive.
Create Inline TLS work request and post to FW.
Signed-off-by: Atul Gupta
---
drivers/crypto/chelsio/chtls/chtls_io.c | 1867 +++
1 file changed, 1867 insertions(+)
create mode 100644 drivers/crypto/chelsio/chtls/chtls_io.c
diff --git a/drivers/crypto/chelsio/chtls/chtls_io.c
b/drivers/crypto/chelsio/chtls/chtls_io.c
new file mode 100644
index 000..a0f03fb
--- /dev/null
+++ b/drivers/crypto/chelsio/chtls/chtls_io.c
@@ -0,0 +1,1867 @@
+/*
+ * Copyright (c) 2017 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Written by: Atul Gupta (atul.gu...@chelsio.com)
+ */
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "chtls.h"
+#include "chtls_cm.h"
+
+static bool is_tls_hw(struct chtls_sock *csk)
+{
+ return csk->tlshws.ofld;
+}
+
+static bool is_tls_rx(struct chtls_sock *csk)
+{
+ return (csk->tlshws.rxkey >= 0);
+}
+
+static bool is_tls_tx(struct chtls_sock *csk)
+{
+ return (csk->tlshws.txkey >= 0);
+}
+
+static bool is_tls_skb(struct chtls_sock *csk, const struct sk_buff *skb)
+{
+ return (is_tls_hw(csk) && skb_ulp_tls_skb_flags(skb));
+}
+
+static int key_size(void *sk)
+{
+ return 16; /* Key on DDR */
+}
+
+#define ceil(x, y) \
+ ({ unsigned long __x = (x), __y = (y); (__x + __y - 1) / __y; })
+
+static int data_sgl_len(const struct sk_buff *skb)
+{
+ unsigned int cnt;
+
+ cnt = skb_shinfo(skb)->nr_frags;
+ return (sgl_len(cnt) * 8);
+}
+
+static int nos_ivs(struct sock *sk, unsigned int size)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+
+ return ceil(size, csk->tlshws.mfs);
+}
+
+#define TLS_WR_CPL_LEN \
+ (sizeof(struct fw_tlstx_data_wr) + \
+ sizeof(struct cpl_tx_tls_sfo))
+
+static int is_ivs_imm(struct sock *sk, const struct sk_buff *skb)
+{
+ int ivs_size = nos_ivs(sk, skb->len) * CIPHER_BLOCK_SIZE;
+ int hlen = TLS_WR_CPL_LEN + data_sgl_len(skb);
+
+ if ((hlen + key_size(sk) + ivs_size) <
+ MAX_IMM_OFLD_TX_DATA_WR_LEN) {
+ ULP_SKB_CB(skb)->ulp.tls.iv = 1;
+ return 1;
+ }
+ ULP_SKB_CB(skb)->ulp.tls.iv = 0;
+ return 0;
+}
+
+static int max_ivs_size(struct sock *sk, int size)
+{
+ return (nos_ivs(sk, size) * CIPHER_BLOCK_SIZE);
+}
+
+static int ivs_size(struct sock *sk, const struct sk_buff *skb)
+{
+ return (is_ivs_imm(sk, skb) ? (nos_ivs(sk, skb->len) *
+CIPHER_BLOCK_SIZE) : 0);
+}
+
+static int flowc_wr_credits(int nparams, int *flowclenp)
+{
+ int flowclen16, flowclen;
+
+ flowclen = offsetof(struct fw_flowc_wr, mnemval[nparams]);
+ flowclen16 = DIV_ROUND_UP(flowclen, 16);
+ flowclen = flowclen16 * 16;
+
+ if (flowclenp)
+ *flowclenp = flowclen;
+
+ return flowclen16;
+}
+
+static struct sk_buff *create_flowc_wr_skb(struct sock *sk,
+ struct fw_flowc_wr *flowc,
+ int flowclen)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+ struct sk_buff *skb;
+
+ skb = alloc_skb(flowclen, GFP_ATOMIC);
+ if (!skb)
+ return NULL;
+
+ memcpy(__skb_put(skb, flowclen), flowc, flowclen);
+ set_queue(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA, sk);
+
+ return skb;
+}
+
+static int send_flowc_wr(struct sock *sk, struct fw_flowc_wr *flowc,
+int flowclen)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+ bool syn_sent = (sk->sk_state == TCP_SYN_SENT);
+ struct tcp_sock *tp = tcp_sk(sk);
+ int flowclen16 = flowclen / 16;
+ struct sk_buff *skb;
+
+ if (csk_flag(sk, CSK_TX_DATA_SENT)) {
+ skb = create_flowc_wr_skb(sk, flowc, flowclen);
+ if (!skb)
+ return -ENOMEM;
+
+ if (syn_sent)
+ __skb_queue_tail(&csk->ooo_queue, skb);
+ else
+ skb_entail(sk, skb,
+ ULPCB_FLAG_NO_HDR | ULPCB_FLAG_NO_APPEND);
+ return 0;
+ }
+
+ if (!syn_sent) {
+ int ret;
+
+ ret = cxgb4_immdata_send(csk->egress_dev,
+csk->txq_idx,
+flowc, flowclen);
+ if (!ret)
+ return flowclen16;
+ }
+ skb = create_flowc_wr_skb(sk, flowc, flowclen);
+ if (!skb)
+ return -
[Crypto v4 09/12] chtls: CPL handler definition
CPL handlers for TLS session, record transmit and receive.
Signed-off-by: Atul Gupta
---
drivers/crypto/chelsio/chtls/chtls_cm.c | 2045 +++
net/ipv4/tcp_minisocks.c|1 +
2 files changed, 2046 insertions(+)
create mode 100644 drivers/crypto/chelsio/chtls/chtls_cm.c
diff --git a/drivers/crypto/chelsio/chtls/chtls_cm.c
b/drivers/crypto/chelsio/chtls/chtls_cm.c
new file mode 100644
index 000..dac3c3a
--- /dev/null
+++ b/drivers/crypto/chelsio/chtls/chtls_cm.c
@@ -0,0 +1,2045 @@
+/*
+ * Copyright (c) 2017 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Written by: Atul Gupta (atul.gu...@chelsio.com)
+ */
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "chtls.h"
+#include "chtls_cm.h"
+
+extern struct request_sock_ops chtls_rsk_ops;
+
+/*
+ * State transitions and actions for close. Note that if we are in SYN_SENT
+ * we remain in that state as we cannot control a connection while it's in
+ * SYN_SENT; such connections are allowed to establish and are then aborted.
+ */
+static unsigned char new_state[16] = {
+ /* current state: new state: action: */
+ /* (Invalid) */ TCP_CLOSE,
+ /* TCP_ESTABLISHED */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
+ /* TCP_SYN_SENT*/ TCP_SYN_SENT,
+ /* TCP_SYN_RECV*/ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
+ /* TCP_FIN_WAIT1 */ TCP_FIN_WAIT1,
+ /* TCP_FIN_WAIT2 */ TCP_FIN_WAIT2,
+ /* TCP_TIME_WAIT */ TCP_CLOSE,
+ /* TCP_CLOSE */ TCP_CLOSE,
+ /* TCP_CLOSE_WAIT */ TCP_LAST_ACK | TCP_ACTION_FIN,
+ /* TCP_LAST_ACK*/ TCP_LAST_ACK,
+ /* TCP_LISTEN */ TCP_CLOSE,
+ /* TCP_CLOSING */ TCP_CLOSING,
+};
+
+static struct chtls_sock *chtls_sock_create(struct chtls_dev *cdev)
+{
+ struct chtls_sock *csk = kzalloc(sizeof(*csk), GFP_ATOMIC);
+
+ if (!csk)
+ return NULL;
+
+ csk->txdata_skb_cache = alloc_skb(TXDATA_SKB_LEN, GFP_ATOMIC);
+ if (!csk->txdata_skb_cache) {
+ kfree(csk);
+ return NULL;
+ }
+
+ kref_init(&csk->kref);
+ csk->cdev = cdev;
+ skb_queue_head_init(&csk->txq);
+ csk->wr_skb_head = NULL;
+ csk->wr_skb_tail = NULL;
+ csk->mss = MAX_MSS;
+ csk->tlshws.ofld = 1;
+ csk->tlshws.txkey = -1;
+ csk->tlshws.rxkey = -1;
+ csk->tlshws.mfs = TLS_MFS;
+ skb_queue_head_init(&csk->tlshws.sk_recv_queue);
+ return csk;
+}
+
+static void chtls_sock_release(struct kref *ref)
+{
+ struct chtls_sock *csk =
+ container_of(ref, struct chtls_sock, kref);
+
+ kfree(csk);
+}
+
+static struct net_device *chtls_ipv4_netdev(struct chtls_dev *cdev,
+ struct sock *sk)
+{
+ struct net_device *ndev = cdev->ports[0];
+
+ if (likely(!inet_sk(sk)->inet_rcv_saddr))
+ return ndev;
+
+ ndev = ip_dev_find(&init_net, inet_sk(sk)->inet_rcv_saddr);
+ if (!ndev)
+ return NULL;
+
+ if (is_vlan_dev(ndev))
+ return vlan_dev_real_dev(ndev);
+ return ndev;
+}
+
+static void assign_rxopt(struct sock *sk, unsigned int opt)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+ struct tcp_sock *tp = tcp_sk(sk);
+ const struct chtls_dev *cdev;
+
+ cdev = csk->cdev;
+ tp->tcp_header_len = sizeof(struct tcphdr);
+ tp->rx_opt.mss_clamp = cdev->mtus[TCPOPT_MSS_G(opt)] - 40;
+ tp->mss_cache= tp->rx_opt.mss_clamp;
+ tp->rx_opt.tstamp_ok = TCPOPT_TSTAMP_G(opt);
+ tp->rx_opt.snd_wscale= TCPOPT_SACK_G(opt);
+ tp->rx_opt.wscale_ok = TCPOPT_WSCALE_OK_G(opt);
+ SND_WSCALE(tp) = TCPOPT_SND_WSCALE_G(opt);
+ if (!tp->rx_opt.wscale_ok)
+ tp->rx_opt.rcv_wscale = 0;
+ if (tp->rx_opt.tstamp_ok) {
+ tp->tcp_header_len += TCPOLEN_TSTAMP_ALIGNED;
+ tp->rx_opt.mss_clamp -= TCPOLEN_TSTAMP_ALIGNED;
+ } else if (csk->opt2 & TSTAMPS_EN_F) {
+ csk->opt2 &= ~TSTAMPS_EN_F;
+ csk->mtu_idx = TCPOPT_MSS_G(opt);
+ }
+}
+
+static void chtls_purge_rcv_queue(struct sock *sk)
+{
+ struct sk_buff *skb;
+
+ while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) {
+ skb_dst_set(skb, (void *)NULL);
+ kfree_skb(skb);
+ }
+}
+
+static void chtls_purge_write_queue(struct s
[Crypto v4 12/12] Makefile Kconfig
Entry for Inline TLS as another driver dependent on cxgb4 and chcr Signed-off-by: Atul Gupta --- drivers/crypto/chelsio/Kconfig| 11 +++ drivers/crypto/chelsio/Makefile | 1 + drivers/crypto/chelsio/chtls/Makefile | 4 3 files changed, 16 insertions(+) create mode 100644 drivers/crypto/chelsio/chtls/Makefile diff --git a/drivers/crypto/chelsio/Kconfig b/drivers/crypto/chelsio/Kconfig index 5ae9f87..930d82d 100644 --- a/drivers/crypto/chelsio/Kconfig +++ b/drivers/crypto/chelsio/Kconfig @@ -29,3 +29,14 @@ config CHELSIO_IPSEC_INLINE default n ---help--- Enable support for IPSec Tx Inline. + +config CRYPTO_DEV_CHELSIO_TLS +tristate "Chelsio Crypto Inline TLS Driver" +depends on CHELSIO_T4 +depends on TLS +select CRYPTO_DEV_CHELSIO +---help--- + Support Chelsio Inline TLS with Chelsio crypto accelerator. + + To compile this driver as a module, choose M here: the module + will be called chtls. diff --git a/drivers/crypto/chelsio/Makefile b/drivers/crypto/chelsio/Makefile index eaecaf1..639e571 100644 --- a/drivers/crypto/chelsio/Makefile +++ b/drivers/crypto/chelsio/Makefile @@ -3,3 +3,4 @@ ccflags-y := -Idrivers/net/ethernet/chelsio/cxgb4 obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chcr.o chcr-objs := chcr_core.o chcr_algo.o chcr-$(CONFIG_CHELSIO_IPSEC_INLINE) += chcr_ipsec.o +obj-$(CONFIG_CRYPTO_DEV_CHELSIO_TLS) += chtls/ diff --git a/drivers/crypto/chelsio/chtls/Makefile b/drivers/crypto/chelsio/chtls/Makefile new file mode 100644 index 000..df13795 --- /dev/null +++ b/drivers/crypto/chelsio/chtls/Makefile @@ -0,0 +1,4 @@ +ccflags-y := -Idrivers/net/ethernet/chelsio/cxgb4 -Idrivers/crypto/chelsio/ + +obj-$(CONFIG_CRYPTO_DEV_CHELSIO_TLS) += chtls.o +chtls-objs := chtls_main.o chtls_cm.o chtls_io.o chtls_hw.o -- 1.8.3.1
[Crypto v4 11/12] chtls: Register the chtls Inline TLS with net tls
Add new uld driver for Inline TLS support. Register ULP for chtls.
Setsockopt to program key on chip. support AES GCM key size 128.
Signed-off-by: Atul Gupta
---
drivers/crypto/chelsio/chtls/chtls_main.c | 619 ++
include/uapi/linux/tls.h | 1 +
2 files changed, 620 insertions(+)
create mode 100644 drivers/crypto/chelsio/chtls/chtls_main.c
diff --git a/drivers/crypto/chelsio/chtls/chtls_main.c
b/drivers/crypto/chelsio/chtls/chtls_main.c
new file mode 100644
index 000..58efb4a
--- /dev/null
+++ b/drivers/crypto/chelsio/chtls/chtls_main.c
@@ -0,0 +1,619 @@
+/*
+ * Copyright (c) 2017 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Written by: Atul Gupta (atul.gu...@chelsio.com)
+ */
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "chtls.h"
+#include "chtls_cm.h"
+
+#define DRV_NAME "chtls"
+
+/*
+ * chtls device management
+ * maintains a list of the chtls devices
+ */
+static LIST_HEAD(cdev_list);
+static DEFINE_MUTEX(cdev_mutex);
+static DEFINE_MUTEX(cdev_list_lock);
+
+static struct proto chtls_base_prot;
+static struct proto chtls_cpl_prot;
+static DEFINE_MUTEX(notify_mutex);
+static RAW_NOTIFIER_HEAD(listen_notify_list);
+struct request_sock_ops chtls_rsk_ops;
+static uint send_page_order = (14 - PAGE_SHIFT < 0) ? 0 : 14 - PAGE_SHIFT;
+
+static int register_listen_notifier(struct notifier_block *nb)
+{
+ int err;
+
+ mutex_lock(¬ify_mutex);
+ err = raw_notifier_chain_register(&listen_notify_list, nb);
+ mutex_unlock(¬ify_mutex);
+ return err;
+}
+
+static int unregister_listen_notifier(struct notifier_block *nb)
+{
+ int err;
+
+ mutex_lock(¬ify_mutex);
+ err = raw_notifier_chain_unregister(&listen_notify_list, nb);
+ mutex_unlock(¬ify_mutex);
+ return err;
+}
+
+static int listen_notify_handler(struct notifier_block *this,
+unsigned long event, void *data)
+{
+ struct sock *sk = data;
+ struct chtls_dev *cdev;
+ int ret = NOTIFY_DONE;
+
+ switch (event) {
+ case CHTLS_LISTEN_START:
+ case CHTLS_LISTEN_STOP:
+ mutex_lock(&cdev_list_lock);
+ list_for_each_entry(cdev, &cdev_list, list) {
+ if (event == CHTLS_LISTEN_START)
+ ret = chtls_listen_start(cdev, sk);
+ else
+ chtls_listen_stop(cdev, sk);
+ }
+ mutex_unlock(&cdev_list_lock);
+ break;
+ }
+ return ret;
+}
+
+static struct notifier_block listen_notifier = {
+ .notifier_call = listen_notify_handler
+};
+
+static int listen_backlog_rcv(struct sock *sk, struct sk_buff *skb)
+{
+ if (likely(skb_transport_header(skb) != skb_network_header(skb)))
+ return tcp_v4_do_rcv(sk, skb);
+ BLOG_SKB_CB(skb)->backlog_rcv(sk, skb);
+ return 0;
+}
+
+static int chtls_start_listen(struct sock *sk)
+{
+ int err;
+
+ if (sk->sk_protocol != IPPROTO_TCP)
+ return -EPROTONOSUPPORT;
+
+ if (sk->sk_family == PF_INET &&
+ LOOPBACK(inet_sk(sk)->inet_rcv_saddr))
+ return -EADDRNOTAVAIL;
+
+ sk->sk_backlog_rcv = listen_backlog_rcv;
+ mutex_lock(¬ify_mutex);
+ err = raw_notifier_call_chain(&listen_notify_list, 0, sk);
+ mutex_unlock(¬ify_mutex);
+ return err;
+}
+
+static int chtls_hash(struct sock *sk)
+{
+ int err;
+
+ err = tcp_prot.hash(sk);
+ if (sk->sk_state == TCP_LISTEN)
+ err |= chtls_start_listen(sk);
+
+ if (err)
+ tcp_prot.unhash(sk);
+ return err;
+}
+
+static int chtls_stop_listen(struct sock *sk)
+{
+ if (sk->sk_protocol != IPPROTO_TCP)
+ return -EPROTONOSUPPORT;
+
+ mutex_lock(¬ify_mutex);
+ raw_notifier_call_chain(&listen_notify_list, 1, sk);
+ mutex_unlock(¬ify_mutex);
+ return 0;
+}
+
+struct net_device *chtls_netdev(struct tls_device *dev,
+ struct net_device *netdev)
+{
+ struct chtls_dev *cdev = to_chtls_dev(dev);
+ int i;
+
+ for (i = 0; i < cdev->lldi->nports; i++)
+ if (cdev->ports[i] == netdev)
+ return netdev;
+
+ return cdev->ports[i];
+}
+
+void chtls_update_prot(struct tls_device *dev, struct sock *sk)
+{
+ sk->sk_prot = &chtls_base_prot;
+}
+
+int chtls_inline_feature(struct tls_device *dev)
+{
+ struct chtls_dev *cdev = to_chtls_dev(dev);
+ struct net_device *netdev;
+ int i;
RE: [Crypto v4 03/12] support for inline tls
-Original Message-
From: David Miller [mailto:da...@davemloft.net]
Sent: Tuesday, February 13, 2018 1:19 AM
To: Atul Gupta
Cc: davejwat...@fb.com; herb...@gondor.apana.org.au; s...@queasysnail.net;
linux-crypto@vger.kernel.org; net...@vger.kernel.org; Ganesh GR
Subject: Re: [Crypto v4 03/12] support for inline tls
From: Atul Gupta
Date: Mon, 12 Feb 2018 17:34:28 +0530
> +static int get_tls_prot(struct sock *sk) {
> + struct tls_context *ctx = tls_get_ctx(sk);
> + struct net_device *netdev;
> + struct tls_device *dev;
> +
> + /* Device bound to specific IP */
> + if (inet_sk(sk)->inet_rcv_saddr) {
> + netdev = find_netdev(sk);
> + if (!netdev)
> + goto out;
> +
> + /* Device supports Inline record processing */
> + if (!(netdev->features & NETIF_F_HW_TLS_INLINE))
> + goto out;
> +
> + mutex_lock(&device_mutex);
> + list_for_each_entry(dev, &device_list, dev_list) {
> + if (dev->netdev && dev->netdev(dev, netdev))
> + break;
> + }
> + mutex_unlock(&device_mutex);
> +
> + ctx->tx_conf = TLS_FULL_HW;
> + if (dev->prot)
> + dev->prot(dev, sk);
What if the same IP address is configured on multiple interfaces?
Thanks, I overlooked this point.
The checks above were based on the premise that device chosen is indeed the one
with Inline TLS enabled, net_device corresponding to specific IP address,
feature configured for device from ethtool and net_device corresponds to Inline
TLS driver registered with net tls.
Case with same IP configured on multiple interface looks similar to INADDR_ANY
below.
The TLS_FULL_HW and modified hash routines handles devices with/without Inline
TLS support. The first Inline TLS capable device updates sk_prot for
TLS_FULL_HW. The tls_hw_hash listens on all interfaces and process device
specific routine, the listen however succeeds for device on which connect is
initiated and may not have the Inline TLS capability, such connection continues
in TLS_BASE_TX or non-tls-offload mode. On the other hand, if Inline TLS
capable device were to establish connection it updates the prot as required for
offload mode to continue.
> + } else { /* src address not known or INADDR_ANY */
> + mutex_lock(&device_mutex);
> + list_for_each_entry(dev, &device_list, dev_list) {
> + if (dev->feature && dev->feature(dev)) {
> + ctx->tx_conf = TLS_FULL_HW;
> + break;
> + }
> + }
> + mutex_unlock(&device_mutex);
> + update_sk_prot(sk, ctx);
And I think this is even more of a stretch. Just because you find an inline
TLS device on the global list doesn't mean traffic will necessarily flow
through it once the connection is fully established and therefore be able to
provide inline TLS offloading.
[Crypto v5 02/12] ethtool: feature for Inline TLS in HW
Signed-off-by: Atul Gupta
---
include/linux/netdev_features.h | 2 ++
net/core/ethtool.c | 1 +
2 files changed, 3 insertions(+)
diff --git a/include/linux/netdev_features.h b/include/linux/netdev_features.h
index b1b0ca7..e1a33b7 100644
--- a/include/linux/netdev_features.h
+++ b/include/linux/netdev_features.h
@@ -77,6 +77,7 @@ enum {
NETIF_F_HW_ESP_BIT, /* Hardware ESP transformation offload
*/
NETIF_F_HW_ESP_TX_CSUM_BIT, /* ESP with TX checksum offload */
NETIF_F_RX_UDP_TUNNEL_PORT_BIT, /* Offload of RX port for UDP tunnels */
+ NETIF_F_HW_TLS_INLINE_BIT, /* Offload TLS record */
/*
* Add your fresh new feature above and remember to update
@@ -142,6 +143,7 @@ enum {
#define NETIF_F_HW_ESP __NETIF_F(HW_ESP)
#define NETIF_F_HW_ESP_TX_CSUM __NETIF_F(HW_ESP_TX_CSUM)
#defineNETIF_F_RX_UDP_TUNNEL_PORT __NETIF_F(RX_UDP_TUNNEL_PORT)
+#define NETIF_F_HW_TLS_INLINE __NETIF_F(HW_TLS_INLINE)
#define for_each_netdev_feature(mask_addr, bit)\
for_each_set_bit(bit, (unsigned long *)mask_addr, NETDEV_FEATURE_COUNT)
diff --git a/net/core/ethtool.c b/net/core/ethtool.c
index f8fcf45..cac1c77 100644
--- a/net/core/ethtool.c
+++ b/net/core/ethtool.c
@@ -106,6 +106,7 @@ int ethtool_op_get_ts_info(struct net_device *dev, struct
ethtool_ts_info *info)
[NETIF_F_HW_ESP_BIT] = "esp-hw-offload",
[NETIF_F_HW_ESP_TX_CSUM_BIT] = "esp-tx-csum-hw-offload",
[NETIF_F_RX_UDP_TUNNEL_PORT_BIT] = "rx-udp_tunnel-port-offload",
+ [NETIF_F_HW_TLS_INLINE_BIT] ="tls-inline",
};
static const char
--
1.8.3.1
[Crypto v5 03/12] support for inline tls
Facility to register Inline TLS drivers to net/tls. Setup
TLS_FULL_HW prot to listen on offload device.
Cases handled
1. Inline TLS device exists, setup prot for TLS_FULL_HW
2. Atleast one Inline TLS exists, sets TLS_FULL_HW. If
non-inline capable device establish connection, move to TLS_SW_TX
3. default mode TLS_SW_TX continues
Signed-off-by: Atul Gupta
---
net/tls/tls_main.c | 124 ++---
1 file changed, 117 insertions(+), 7 deletions(-)
diff --git a/net/tls/tls_main.c b/net/tls/tls_main.c
index e07ee3a..81c61e9 100644
--- a/net/tls/tls_main.c
+++ b/net/tls/tls_main.c
@@ -38,6 +38,7 @@
#include
#include
#include
+#include
#include
@@ -45,13 +46,9 @@
MODULE_DESCRIPTION("Transport Layer Security Support");
MODULE_LICENSE("Dual BSD/GPL");
-enum {
- TLS_BASE_TX,
- TLS_SW_TX,
- TLS_NUM_CONFIG,
-};
-
-static struct proto tls_prots[TLS_NUM_CONFIG];
+static LIST_HEAD(device_list);
+static DEFINE_MUTEX(device_mutex);
+struct proto tls_prots[TLS_NUM_CONFIG];
static inline void update_sk_prot(struct sock *sk, struct tls_context *ctx)
{
@@ -260,6 +257,38 @@ static void tls_sk_proto_close(struct sock *sk, long
timeout)
sk_proto_close(sk, timeout);
}
+static struct net_device *get_netdev(struct sock *sk)
+{
+ struct inet_sock *inet = inet_sk(sk);
+ struct net_device *netdev = NULL;
+
+ netdev = dev_get_by_index(sock_net(sk), inet->cork.fl.flowi_oif);
+ return netdev;
+}
+
+static int get_tls_offload_dev(struct sock *sk)
+{
+ struct net_device *netdev;
+ struct tls_device *dev;
+ int rc = -EINVAL;
+
+ netdev = get_netdev(sk);
+ if (!netdev)
+ goto out;
+
+ mutex_lock(&device_mutex);
+ list_for_each_entry(dev, &device_list, dev_list) {
+ if (dev->netdev && dev->netdev(dev, netdev)) {
+ rc = 0;
+ break;
+ }
+ }
+ mutex_unlock(&device_mutex);
+ dev_put(netdev);
+out:
+ return rc;
+}
+
static int do_tls_getsockopt_tx(struct sock *sk, char __user *optval,
int __user *optlen)
{
@@ -401,6 +430,15 @@ static int do_tls_setsockopt_tx(struct sock *sk, char
__user *optval,
goto out;
}
+ rc = get_tls_offload_dev(sk);
+ if (rc) {
+ goto out;
+ } else {
+ /* Retain HW unhash for cleanup and move to SW Tx */
+ sk->sk_prot[TLS_BASE_TX].unhash =
+ sk->sk_prot[TLS_FULL_HW].unhash;
+ }
+
/* currently SW is default, we will have ethtool in future */
rc = tls_set_sw_offload(sk, ctx);
tx_conf = TLS_SW_TX;
@@ -448,6 +486,54 @@ static int tls_setsockopt(struct sock *sk, int level, int
optname,
return do_tls_setsockopt(sk, optname, optval, optlen);
}
+static int tls_hw_prot(struct sock *sk)
+{
+ struct tls_context *ctx = tls_get_ctx(sk);
+ struct tls_device *dev;
+
+ mutex_lock(&device_mutex);
+ list_for_each_entry(dev, &device_list, dev_list) {
+ if (dev->feature && dev->feature(dev)) {
+ ctx->tx_conf = TLS_FULL_HW;
+ update_sk_prot(sk, ctx);
+ break;
+ }
+ }
+ mutex_unlock(&device_mutex);
+ return ctx->tx_conf;
+}
+
+static void tls_hw_unhash(struct sock *sk)
+{
+ struct tls_device *dev;
+
+ mutex_lock(&device_mutex);
+ list_for_each_entry(dev, &device_list, dev_list) {
+ if (dev->unhash)
+ dev->unhash(dev, sk);
+ }
+ mutex_unlock(&device_mutex);
+ tcp_prot.unhash(sk);
+}
+
+static int tls_hw_hash(struct sock *sk)
+{
+ struct tls_device *dev;
+ int err;
+
+ err = tcp_prot.hash(sk);
+ mutex_lock(&device_mutex);
+ list_for_each_entry(dev, &device_list, dev_list) {
+ if (dev->hash)
+ err |= dev->hash(dev, sk);
+ }
+ mutex_unlock(&device_mutex);
+
+ if (err)
+ tls_hw_unhash(sk);
+ return err;
+}
+
static int tls_init(struct sock *sk)
{
struct inet_connection_sock *icsk = inet_csk(sk);
@@ -466,6 +552,9 @@ static int tls_init(struct sock *sk)
ctx->sk_proto_close = sk->sk_prot->close;
ctx->tx_conf = TLS_BASE_TX;
+ if (tls_hw_prot(sk) == TLS_FULL_HW)
+ goto out;
+
update_sk_prot(sk, ctx);
out:
return rc;
@@ -487,7 +576,27 @@ static void build_protos(struct proto *prot, struct proto
*base)
prot[TLS_SW_TX] = prot[TLS_BASE_TX];
prot[TLS_SW_TX].sendmsg = tls_sw_sendmsg;
prot[TLS_SW_TX].sendpage= tls_sw_sendpage;
+
+ prot[TLS_FULL_HW] = prot[T
[Crypto v5 04/12] chtls: structure and macro definiton
Inline TLS state, connection management. Supporting macros definition.
Signed-off-by: Atul Gupta
---
drivers/crypto/chelsio/chtls/chtls.h| 487
drivers/crypto/chelsio/chtls/chtls_cm.h | 203 +
2 files changed, 690 insertions(+)
create mode 100644 drivers/crypto/chelsio/chtls/chtls.h
create mode 100644 drivers/crypto/chelsio/chtls/chtls_cm.h
diff --git a/drivers/crypto/chelsio/chtls/chtls.h
b/drivers/crypto/chelsio/chtls/chtls.h
new file mode 100644
index 000..c7b8d59
--- /dev/null
+++ b/drivers/crypto/chelsio/chtls/chtls.h
@@ -0,0 +1,487 @@
+/*
+ * Copyright (c) 2016 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef __CHTLS_H__
+#define __CHTLS_H__
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "t4fw_api.h"
+#include "t4_msg.h"
+#include "cxgb4.h"
+#include "cxgb4_uld.h"
+#include "l2t.h"
+#include "chcr_algo.h"
+#include "chcr_core.h"
+#include "chcr_crypto.h"
+
+#define CIPHER_BLOCK_SIZE 16
+#define MAX_IVS_PAGE256
+#define TLS_KEY_CONTEXT_SZ 64
+#define TLS_HEADER_LENGTH 5
+#define SCMD_CIPH_MODE_AES_GCM 2
+#define GCM_TAG_SIZE16
+#define AEAD_EXPLICIT_DATA_SIZE 8
+/* Any MFS size should work and come from openssl */
+#define TLS_MFS16384
+
+#define SOCK_INLINE (31)
+#define RSS_HDR sizeof(struct rss_header)
+
+enum {
+ CHTLS_KEY_CONTEXT_DSGL,
+ CHTLS_KEY_CONTEXT_IMM,
+ CHTLS_KEY_CONTEXT_DDR,
+};
+
+enum {
+ CHTLS_LISTEN_START,
+ CHTLS_LISTEN_STOP,
+};
+
+/* Flags for return value of CPL message handlers */
+enum {
+ CPL_RET_BUF_DONE = 1, /* buffer processing done */
+ CPL_RET_BAD_MSG = 2,/* bad CPL message */
+ CPL_RET_UNKNOWN_TID = 4 /* unexpected unknown TID */
+};
+
+#define TLS_RCV_ST_READ_HEADER 0xF0
+#define TLS_RCV_ST_READ_BODY0xF1
+#define TLS_RCV_ST_READ_DONE0xF2
+#define TLS_RCV_ST_READ_NB 0xF3
+
+#define RSPQ_HASH_BITS 5
+#define LISTEN_INFO_HASH_SIZE 32
+struct listen_info {
+ struct listen_info *next; /* Link to next entry */
+ struct sock *sk; /* The listening socket */
+ unsigned int stid; /* The server TID */
+};
+
+enum {
+ T4_LISTEN_START_PENDING,
+ T4_LISTEN_STARTED
+};
+
+enum csk_flags {
+ CSK_CALLBACKS_CHKD, /* socket callbacks have been sanitized */
+ CSK_ABORT_REQ_RCVD, /* received one ABORT_REQ_RSS message */
+ CSK_TX_MORE_DATA, /* sending ULP data; don't set SHOVE bit */
+ CSK_TX_WAIT_IDLE, /* suspend Tx until in-flight data is ACKed */
+ CSK_ABORT_SHUTDOWN, /* shouldn't send more abort requests */
+ CSK_ABORT_RPL_PENDING, /* expecting an abort reply */
+ CSK_CLOSE_CON_REQUESTED,/* we've sent a close_conn_req */
+ CSK_TX_DATA_SENT, /* sent a TX_DATA WR on this connection */
+ CSK_TX_FAILOVER,/* Tx traffic failing over */
+ CSK_UPDATE_RCV_WND, /* Need to update rcv window */
+ CSK_RST_ABORTED,/* outgoing RST was aborted */
+ CSK_TLS_HANDSHK,/* TLS Handshake */
+};
+
+struct listen_ctx {
+ struct sock *lsk;
+ struct chtls_dev *cdev;
+ u32 state;
+};
+
+struct key_map {
+ unsigned long *addr;
+ unsigned int start;
+ unsigned int available;
+ unsigned int size;
+ spinlock_t lock; /* lock for key id request from map */
+} __packed;
+
+struct tls_scmd {
+ __be32 seqno_numivs;
+ __be32 ivgen_hdrlen;
+};
+
+struct chtls_dev {
+ struct tls_device tlsdev;
+ struct list_head list;
+ struct cxgb4_lld_info *lldi;
+ struct pci_dev *pdev;
+ struct listen_info *listen_hash_tab[LISTEN_INFO_HASH_SIZE];
+ spinlock_t listen_lock; /* lock for listen list */
+ struct net_device **ports;
+ struct tid_info *tids;
+ unsigned int pfvf;
+ const unsigned short *mtus;
+
+ spinlock_t aidr_lock cacheline_aligned_in_smp;
+ struct idr aidr; /* ATID id space */
+ struct idr hwtid_idr;
+ struct idr stid_idr;
+
+ spinlock_t idr_lock cacheline_aligned_in_smp;
+
+ struct net_device *egr_dev[NCHAN * 2];
+ struct sk_buff *rspq_skb_cache[1 << RSPQ_HASH_BITS];
+ struct sk_buff *askb;
+
+ struct sk_buff_head deferq;
+ struct work_struct deferq_task;
+
+ struct list_head list_node;
+ struct list_head rcu_node;
+ struct list_head na_node;
+
[Crypto v5 01/12] tls: tls_device struct to register TLS drivers
tls_device structure to register Inline TLS drivers
with net/tls
Signed-off-by: Atul Gupta
---
include/net/tls.h | 24
1 file changed, 24 insertions(+)
diff --git a/include/net/tls.h b/include/net/tls.h
index 936cfc5..6b64510 100644
--- a/include/net/tls.h
+++ b/include/net/tls.h
@@ -55,6 +55,28 @@
#define TLS_AAD_SPACE_SIZE 13
+#define TLS_DEVICE_NAME_MAX 32
+
+enum {
+ TLS_BASE_TX,
+ TLS_SW_TX,
+ TLS_FULL_HW, /* TLS record processed Inline */
+ TLS_NUM_CONFIG,
+};
+extern struct proto tls_prots[TLS_NUM_CONFIG];
+
+struct tls_device {
+ char name[TLS_DEVICE_NAME_MAX];
+ struct list_head dev_list;
+
+ /* netdev present in registered inline tls driver */
+ int (*netdev)(struct tls_device *device,
+ struct net_device *netdev);
+ int (*feature)(struct tls_device *device);
+ int (*hash)(struct tls_device *device, struct sock *sk);
+ void (*unhash)(struct tls_device *device, struct sock *sk);
+};
+
struct tls_sw_context {
struct crypto_aead *aead_send;
@@ -254,5 +276,7 @@ static inline struct tls_offload_context *tls_offload_ctx(
int tls_proccess_cmsg(struct sock *sk, struct msghdr *msg,
unsigned char *record_type);
+void tls_register_device(struct tls_device *device);
+void tls_unregister_device(struct tls_device *device);
#endif /* _TLS_OFFLOAD_H */
--
1.8.3.1
[Crypto v5 09/12] chtls: CPL handler definition
CPL handlers for TLS session, record transmit and receive.
Signed-off-by: Atul Gupta
---
drivers/crypto/chelsio/chtls/chtls_cm.c | 2046 +++
net/ipv4/tcp_minisocks.c|1 +
2 files changed, 2047 insertions(+)
create mode 100644 drivers/crypto/chelsio/chtls/chtls_cm.c
diff --git a/drivers/crypto/chelsio/chtls/chtls_cm.c
b/drivers/crypto/chelsio/chtls/chtls_cm.c
new file mode 100644
index 000..670bac6
--- /dev/null
+++ b/drivers/crypto/chelsio/chtls/chtls_cm.c
@@ -0,0 +1,2046 @@
+/*
+ * Copyright (c) 2017 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Written by: Atul Gupta (atul.gu...@chelsio.com)
+ */
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "chtls.h"
+#include "chtls_cm.h"
+
+extern struct request_sock_ops chtls_rsk_ops;
+
+/*
+ * State transitions and actions for close. Note that if we are in SYN_SENT
+ * we remain in that state as we cannot control a connection while it's in
+ * SYN_SENT; such connections are allowed to establish and are then aborted.
+ */
+static unsigned char new_state[16] = {
+ /* current state: new state: action: */
+ /* (Invalid) */ TCP_CLOSE,
+ /* TCP_ESTABLISHED */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
+ /* TCP_SYN_SENT*/ TCP_SYN_SENT,
+ /* TCP_SYN_RECV*/ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
+ /* TCP_FIN_WAIT1 */ TCP_FIN_WAIT1,
+ /* TCP_FIN_WAIT2 */ TCP_FIN_WAIT2,
+ /* TCP_TIME_WAIT */ TCP_CLOSE,
+ /* TCP_CLOSE */ TCP_CLOSE,
+ /* TCP_CLOSE_WAIT */ TCP_LAST_ACK | TCP_ACTION_FIN,
+ /* TCP_LAST_ACK*/ TCP_LAST_ACK,
+ /* TCP_LISTEN */ TCP_CLOSE,
+ /* TCP_CLOSING */ TCP_CLOSING,
+};
+
+static struct chtls_sock *chtls_sock_create(struct chtls_dev *cdev)
+{
+ struct chtls_sock *csk = kzalloc(sizeof(*csk), GFP_ATOMIC);
+
+ if (!csk)
+ return NULL;
+
+ csk->txdata_skb_cache = alloc_skb(TXDATA_SKB_LEN, GFP_ATOMIC);
+ if (!csk->txdata_skb_cache) {
+ kfree(csk);
+ return NULL;
+ }
+
+ kref_init(&csk->kref);
+ csk->cdev = cdev;
+ skb_queue_head_init(&csk->txq);
+ csk->wr_skb_head = NULL;
+ csk->wr_skb_tail = NULL;
+ csk->mss = MAX_MSS;
+ csk->tlshws.ofld = 1;
+ csk->tlshws.txkey = -1;
+ csk->tlshws.rxkey = -1;
+ csk->tlshws.mfs = TLS_MFS;
+ skb_queue_head_init(&csk->tlshws.sk_recv_queue);
+ return csk;
+}
+
+static void chtls_sock_release(struct kref *ref)
+{
+ struct chtls_sock *csk =
+ container_of(ref, struct chtls_sock, kref);
+
+ kfree(csk);
+}
+
+static struct net_device *chtls_ipv4_netdev(struct chtls_dev *cdev,
+ struct sock *sk)
+{
+ struct net_device *ndev = cdev->ports[0];
+
+ if (likely(!inet_sk(sk)->inet_rcv_saddr))
+ return ndev;
+
+ ndev = ip_dev_find(&init_net, inet_sk(sk)->inet_rcv_saddr);
+ if (!ndev)
+ return NULL;
+
+ if (is_vlan_dev(ndev))
+ return vlan_dev_real_dev(ndev);
+ return ndev;
+}
+
+static void assign_rxopt(struct sock *sk, unsigned int opt)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+ struct tcp_sock *tp = tcp_sk(sk);
+ const struct chtls_dev *cdev;
+
+ cdev = csk->cdev;
+ tp->tcp_header_len = sizeof(struct tcphdr);
+ tp->rx_opt.mss_clamp = cdev->mtus[TCPOPT_MSS_G(opt)] - 40;
+ tp->mss_cache= tp->rx_opt.mss_clamp;
+ tp->rx_opt.tstamp_ok = TCPOPT_TSTAMP_G(opt);
+ tp->rx_opt.snd_wscale= TCPOPT_SACK_G(opt);
+ tp->rx_opt.wscale_ok = TCPOPT_WSCALE_OK_G(opt);
+ SND_WSCALE(tp) = TCPOPT_SND_WSCALE_G(opt);
+ if (!tp->rx_opt.wscale_ok)
+ tp->rx_opt.rcv_wscale = 0;
+ if (tp->rx_opt.tstamp_ok) {
+ tp->tcp_header_len += TCPOLEN_TSTAMP_ALIGNED;
+ tp->rx_opt.mss_clamp -= TCPOLEN_TSTAMP_ALIGNED;
+ } else if (csk->opt2 & TSTAMPS_EN_F) {
+ csk->opt2 &= ~TSTAMPS_EN_F;
+ csk->mtu_idx = TCPOPT_MSS_G(opt);
+ }
+}
+
+static void chtls_purge_rcv_queue(struct sock *sk)
+{
+ struct sk_buff *skb;
+
+ while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) {
+ skb_dst_set(skb, (void *)NULL);
+ kfree_skb(skb);
+ }
+}
+
+static void chtls_purge_write_queue(struct s
[Crypto v5 00/12] Chelsio Inline TLS
Series for Chelsio Inline TLS driver (chtls.ko)
Driver use the ULP infrastructure to register chtls as Inline TLS ULP.
Chtls use TCP Sockets to transmit and receive TLS record. TCP proto_ops
is extended to offload TLS record.
T6 adapter provides the following features:
-TLS record offload, TLS header, encrypt, digest and transmit
-TLS record receive and decrypt
-TLS keys store
-TCP/IP engine
-TLS engine
-GCM crypto engine [support CBC also]
TLS provides security at the transport layer. It uses TCP to provide
reliable end-to-end transport of application data. It relies on TCP
for any retransmission. TLS session comprises of three parts:
a. TCP/IP connection
b. TLS handshake
c. Record layer processing
TLS handshake state machine is executed in host (refer standard
implementation eg. OpenSSL). Setsockopt [SOL_TCP, TCP_ULP] initialize
TCP proto-ops for Chelsio inline tls support. setsockopt(sock, SOL_TCP,
TCP_ULP, "tls", sizeof("tls"));
Tx and Rx Keys are decided during handshake and programmed onto the chip
after CCS is exchanged.
struct tls12_crypto_info_aes_gcm_128 crypto_info
setsockopt(sock, SOL_TLS, TLS_TX, &crypto_info, sizeof(crypto_info))
Finish is the first encrypted/decrypted message tx/rx inline.
On the Tx path TLS engine receive plain text from openssl, insert IV,
fetches the tx key, create cipher text records and generate MAC. TLS
header is added to cipher text and forward to TCP/IP engine for transport
layer processing and transmission on wire.
TX:
Application--openssl--chtls---TLS engine---encrypt/auth---TCP/IP
engine---wire.
On the Rx side, data received is PDU aligned at record
boundaries. TLS processes only the complete record. If rx key is programmed
on CCS receive, data is decrypted and plain text is posted to host.
RX:
Wire--cipher-text--TCP/IP engine [PDU align]---TLS engine---
decrypt/auth---plain-text--chtls--openssl--application
v5: set TLS_FULL_HW for registered inline tls drivers
-set TLS_FULL_HW prot for offload connection else move
to TLS_SW_TX
-Case handled for interface with same IP [David Miller]
-Removed Specific IP and INADDR_ANY handling [v4]
v4: removed chtls ULP type, retained tls ULP
-registered chtls with net tls
-defined struct tls_device to register the Inline drivers
-ethtool interface tls-inline to enable Inline TLS for interface
-prot update to support inline TLS
v3: fixed the kbuild test issues
-made few funtions static
-initialized few variables
v2: fixed the following based on the review comments of Stephan Mueller,
Stefano Brivio and Hannes Frederic
-Added more details in cover letter
-Fixed indentation and formating issues
-Using aes instead of aes-generic
-memset key info after programing the key on chip
-reordered the patch sequence
Atul Gupta (12):
tls: tls_device struct to register TLS drivers
ethtool: feature for Inline TLS in HW
support for inline tls
chtls: structure and macro definiton
cxgb4: Inline TLS FW Interface
cxgb4: LLD driver changes to enable TLS
chcr: Key Macro
chtls: Key program
chtls: CPL handler definition
chtls: Inline crypto request Tx/Rx
chtls: Register the chtls Inline TLS with net tls
Makefile Kconfig
drivers/crypto/chelsio/Kconfig | 11 +
drivers/crypto/chelsio/Makefile|1 +
drivers/crypto/chelsio/chcr_algo.h | 42 +
drivers/crypto/chelsio/chcr_core.h | 55 +-
drivers/crypto/chelsio/chtls/Makefile |4 +
drivers/crypto/chelsio/chtls/chtls.h | 487 +
drivers/crypto/chelsio/chtls/chtls_cm.c| 2046
drivers/crypto/chelsio/chtls/chtls_cm.h| 203 ++
drivers/crypto/chelsio/chtls/chtls_hw.c| 394
drivers/crypto/chelsio/chtls/chtls_io.c| 1867 ++
drivers/crypto/chelsio/chtls/chtls_main.c | 574 ++
drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c | 18 +-
drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c| 32 +-
drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h |7 +
drivers/net/ethernet/chelsio/cxgb4/sge.c | 98 +-
drivers/net/ethernet/chelsio/cxgb4/t4_msg.h| 121 +-
drivers/net/ethernet/chelsio/cxgb4/t4_regs.h |2 +
drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h | 165 +-
include/linux/netdev_features.h|2 +
include/net/tls.h | 24 +
include/uapi/linux/tls.h |1 +
net/core/ethtool.c |1 +
net/ipv4/tcp_minisocks.c |1 +
net/tls/tls_main.c | 124 +-
24 files changed, 6254 insertions(+), 26 deletions(-)
create mode 100644 drivers/crypto/chelsio/chtls/Makefile
create mode 100644 drivers/crypto/chelsio/chtls/chtls.h
create mo
[Crypto v5 06/12] cxgb4: LLD driver changes to enable TLS
Read FW capability. Read key area size. Dump the TLS record count.
Signed-off-by: Atul Gupta
---
drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c | 18 +++-
drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c| 32 +--
drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h | 7 ++
drivers/net/ethernet/chelsio/cxgb4/sge.c | 98 +-
4 files changed, 142 insertions(+), 13 deletions(-)
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
index cf47183..cfc9210 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
@@ -2826,8 +2826,8 @@ static int meminfo_show(struct seq_file *seq, void *v)
"Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
"TDDP region:", "TPT region:", "STAG region:", "RQ region:",
"RQUDP region:", "PBL region:", "TXPBL region:",
- "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
- "On-chip queues:"
+ "TLSKey region:", "DBVFIFO region:", "ULPRX state:",
+ "ULPTX state:", "On-chip queues:"
};
int i, n;
@@ -2943,6 +2943,12 @@ static int meminfo_show(struct seq_file *seq, void *v)
ulp_region(RX_RQUDP);
ulp_region(RX_PBL);
ulp_region(TX_PBL);
+ if (adap->params.crypto & FW_CAPS_CONFIG_TLS_INLINE) {
+ ulp_region(RX_TLS_KEY);
+ } else {
+ md->base = 0;
+ md->idx = ARRAY_SIZE(region);
+ }
#undef ulp_region
md->base = 0;
md->idx = ARRAY_SIZE(region);
@@ -3098,6 +3104,14 @@ static int chcr_show(struct seq_file *seq, void *v)
atomic_read(&adap->chcr_stats.fallback));
seq_printf(seq, "IPSec PDU: %10u\n",
atomic_read(&adap->chcr_stats.ipsec_cnt));
+
+ seq_puts(seq, "\nChelsio Inline TLS Stats\n");
+ seq_printf(seq, "TLS PDU Tx: %u\n",
+ atomic_read(&adap->chcr_stats.tls_pdu_tx));
+ seq_printf(seq, "TLS PDU Rx: %u\n",
+ atomic_read(&adap->chcr_stats.tls_pdu_rx));
+ seq_printf(seq, "TLS Keys (DDR) Count: %u\n",
+ atomic_read(&adap->chcr_stats.tls_key));
return 0;
}
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
index 05a4abf..60eb18b 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
@@ -4086,18 +4086,32 @@ static int adap_init0(struct adapter *adap)
adap->num_ofld_uld += 2;
}
if (caps_cmd.cryptocaps) {
- /* Should query params here...TODO */
- params[0] = FW_PARAM_PFVF(NCRYPTO_LOOKASIDE);
- ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2,
- params, val);
- if (ret < 0) {
- if (ret != -EINVAL)
+ if (ntohs(caps_cmd.cryptocaps) &
+ FW_CAPS_CONFIG_CRYPTO_LOOKASIDE) {
+ params[0] = FW_PARAM_PFVF(NCRYPTO_LOOKASIDE);
+ ret = t4_query_params(adap, adap->mbox, adap->pf, 0,
+ 2, params, val);
+ if (ret < 0) {
+ if (ret != -EINVAL)
+ goto bye;
+ } else {
+ adap->vres.ncrypto_fc = val[0];
+ }
+ adap->num_ofld_uld += 1;
+ }
+ if (ntohs(caps_cmd.cryptocaps) &
+ FW_CAPS_CONFIG_TLS_INLINE) {
+ params[0] = FW_PARAM_PFVF(TLS_START);
+ params[1] = FW_PARAM_PFVF(TLS_END);
+ ret = t4_query_params(adap, adap->mbox, adap->pf, 0,
+ 2, params, val);
+ if (ret < 0)
goto bye;
- } else {
- adap->vres.ncrypto_fc = val[0];
+ adap->vres.key.start = val[0];
+ adap->vres.key.size = val[1] - val[0] + 1;
+ adap->num_uld += 1;
}
adap->params.crypto = ntohs(caps_cmd.cryptocaps);
- adap->num_uld += 1;
}
#undef FW_PARAM_PFVF
#undef FW_PARAM_DEV
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h
b/drivers/
[Crypto v5 11/12] chtls: Register the chtls Inline TLS with net tls
Add new uld driver for Inline TLS support. Register ULP for chtls.
Setsockopt to program key on chip. support AES GCM key size 128.
Signed-off-by: Atul Gupta
---
drivers/crypto/chelsio/chtls/chtls_main.c | 574 ++
include/uapi/linux/tls.h | 1 +
2 files changed, 575 insertions(+)
create mode 100644 drivers/crypto/chelsio/chtls/chtls_main.c
diff --git a/drivers/crypto/chelsio/chtls/chtls_main.c
b/drivers/crypto/chelsio/chtls/chtls_main.c
new file mode 100644
index 000..66d4ce9
--- /dev/null
+++ b/drivers/crypto/chelsio/chtls/chtls_main.c
@@ -0,0 +1,574 @@
+/*
+ * Copyright (c) 2017 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Written by: Atul Gupta (atul.gu...@chelsio.com)
+ */
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "chtls.h"
+#include "chtls_cm.h"
+
+#define DRV_NAME "chtls"
+
+/*
+ * chtls device management
+ * maintains a list of the chtls devices
+ */
+static LIST_HEAD(cdev_list);
+static DEFINE_MUTEX(cdev_mutex);
+static DEFINE_MUTEX(cdev_list_lock);
+
+static struct proto chtls_cpl_prot;
+static DEFINE_MUTEX(notify_mutex);
+static RAW_NOTIFIER_HEAD(listen_notify_list);
+struct request_sock_ops chtls_rsk_ops;
+static uint send_page_order = (14 - PAGE_SHIFT < 0) ? 0 : 14 - PAGE_SHIFT;
+
+static int register_listen_notifier(struct notifier_block *nb)
+{
+ int err;
+
+ mutex_lock(¬ify_mutex);
+ err = raw_notifier_chain_register(&listen_notify_list, nb);
+ mutex_unlock(¬ify_mutex);
+ return err;
+}
+
+static int unregister_listen_notifier(struct notifier_block *nb)
+{
+ int err;
+
+ mutex_lock(¬ify_mutex);
+ err = raw_notifier_chain_unregister(&listen_notify_list, nb);
+ mutex_unlock(¬ify_mutex);
+ return err;
+}
+
+static int listen_notify_handler(struct notifier_block *this,
+unsigned long event, void *data)
+{
+ struct sock *sk = data;
+ struct chtls_dev *cdev;
+ int ret = NOTIFY_DONE;
+
+ switch (event) {
+ case CHTLS_LISTEN_START:
+ case CHTLS_LISTEN_STOP:
+ mutex_lock(&cdev_list_lock);
+ list_for_each_entry(cdev, &cdev_list, list) {
+ if (event == CHTLS_LISTEN_START)
+ ret = chtls_listen_start(cdev, sk);
+ else
+ chtls_listen_stop(cdev, sk);
+ }
+ mutex_unlock(&cdev_list_lock);
+ break;
+ }
+ return ret;
+}
+
+static struct notifier_block listen_notifier = {
+ .notifier_call = listen_notify_handler
+};
+
+static int listen_backlog_rcv(struct sock *sk, struct sk_buff *skb)
+{
+ if (likely(skb_transport_header(skb) != skb_network_header(skb)))
+ return tcp_v4_do_rcv(sk, skb);
+ BLOG_SKB_CB(skb)->backlog_rcv(sk, skb);
+ return 0;
+}
+
+static int chtls_start_listen(struct sock *sk)
+{
+ int err;
+
+ if (sk->sk_protocol != IPPROTO_TCP)
+ return -EPROTONOSUPPORT;
+
+ if (sk->sk_family == PF_INET &&
+ LOOPBACK(inet_sk(sk)->inet_rcv_saddr))
+ return -EADDRNOTAVAIL;
+
+ sk->sk_backlog_rcv = listen_backlog_rcv;
+ mutex_lock(¬ify_mutex);
+ err = raw_notifier_call_chain(&listen_notify_list, 0, sk);
+ mutex_unlock(¬ify_mutex);
+ return err;
+}
+
+static int chtls_stop_listen(struct sock *sk)
+{
+ if (sk->sk_protocol != IPPROTO_TCP)
+ return -EPROTONOSUPPORT;
+
+ mutex_lock(¬ify_mutex);
+ raw_notifier_call_chain(&listen_notify_list, 1, sk);
+ mutex_unlock(¬ify_mutex);
+ return 0;
+}
+
+int chtls_netdev(struct tls_device *dev,
+struct net_device *netdev)
+{
+ struct chtls_dev *cdev = to_chtls_dev(dev);
+ int i;
+
+ for (i = 0; i < cdev->lldi->nports; i++)
+ if (cdev->ports[i] == netdev)
+ return 1;
+
+ return 0;
+}
+
+int chtls_inline_feature(struct tls_device *dev)
+{
+ struct chtls_dev *cdev = to_chtls_dev(dev);
+ struct net_device *netdev;
+ int i;
+
+ for (i = 0; i < cdev->lldi->nports; i++) {
+ netdev = cdev->ports[i];
+ if (netdev->features & NETIF_F_HW_TLS_INLINE)
+ return 1;
+ }
+ return 1;
+}
+
+int chtls_create_hash(struct tls_device *dev, struct sock *sk)
+{
+ if (sk->sk_state == TCP_LISTEN)
+ return chtls_start_listen(sk);
+ return 0;
+}
+
+void chtls_destroy_hash(struct tls_device *
[Crypto v5 07/12] chcr: Key Macro
Define macro for TLS Key context
Signed-off-by: Atul Gupta
---
drivers/crypto/chelsio/chcr_algo.h | 42 +
drivers/crypto/chelsio/chcr_core.h | 55 +-
2 files changed, 96 insertions(+), 1 deletion(-)
diff --git a/drivers/crypto/chelsio/chcr_algo.h
b/drivers/crypto/chelsio/chcr_algo.h
index d1673a5..f263cd4 100644
--- a/drivers/crypto/chelsio/chcr_algo.h
+++ b/drivers/crypto/chelsio/chcr_algo.h
@@ -86,6 +86,39 @@
KEY_CONTEXT_OPAD_PRESENT_M)
#define KEY_CONTEXT_OPAD_PRESENT_F KEY_CONTEXT_OPAD_PRESENT_V(1U)
+#define TLS_KEYCTX_RXFLIT_CNT_S 24
+#define TLS_KEYCTX_RXFLIT_CNT_V(x) ((x) << TLS_KEYCTX_RXFLIT_CNT_S)
+
+#define TLS_KEYCTX_RXPROT_VER_S 20
+#define TLS_KEYCTX_RXPROT_VER_M 0xf
+#define TLS_KEYCTX_RXPROT_VER_V(x) ((x) << TLS_KEYCTX_RXPROT_VER_S)
+
+#define TLS_KEYCTX_RXCIPH_MODE_S 16
+#define TLS_KEYCTX_RXCIPH_MODE_M 0xf
+#define TLS_KEYCTX_RXCIPH_MODE_V(x) ((x) << TLS_KEYCTX_RXCIPH_MODE_S)
+
+#define TLS_KEYCTX_RXAUTH_MODE_S 12
+#define TLS_KEYCTX_RXAUTH_MODE_M 0xf
+#define TLS_KEYCTX_RXAUTH_MODE_V(x) ((x) << TLS_KEYCTX_RXAUTH_MODE_S)
+
+#define TLS_KEYCTX_RXCIAU_CTRL_S 11
+#define TLS_KEYCTX_RXCIAU_CTRL_V(x) ((x) << TLS_KEYCTX_RXCIAU_CTRL_S)
+
+#define TLS_KEYCTX_RX_SEQCTR_S 9
+#define TLS_KEYCTX_RX_SEQCTR_M 0x3
+#define TLS_KEYCTX_RX_SEQCTR_V(x) ((x) << TLS_KEYCTX_RX_SEQCTR_S)
+
+#define TLS_KEYCTX_RX_VALID_S 8
+#define TLS_KEYCTX_RX_VALID_V(x) ((x) << TLS_KEYCTX_RX_VALID_S)
+
+#define TLS_KEYCTX_RXCK_SIZE_S 3
+#define TLS_KEYCTX_RXCK_SIZE_M 0x7
+#define TLS_KEYCTX_RXCK_SIZE_V(x) ((x) << TLS_KEYCTX_RXCK_SIZE_S)
+
+#define TLS_KEYCTX_RXMK_SIZE_S 0
+#define TLS_KEYCTX_RXMK_SIZE_M 0x7
+#define TLS_KEYCTX_RXMK_SIZE_V(x) ((x) << TLS_KEYCTX_RXMK_SIZE_S)
+
#define CHCR_HASH_MAX_DIGEST_SIZE 64
#define CHCR_MAX_SHA_DIGEST_SIZE 64
@@ -176,6 +209,15 @@
KEY_CONTEXT_SALT_PRESENT_V(1) | \
KEY_CONTEXT_CTX_LEN_V((ctx_len)))
+#define FILL_KEY_CRX_HDR(ck_size, mk_size, d_ck, opad, ctx_len) \
+ htonl(TLS_KEYCTX_RXMK_SIZE_V(mk_size) | \
+ TLS_KEYCTX_RXCK_SIZE_V(ck_size) | \
+ TLS_KEYCTX_RX_VALID_V(1) | \
+ TLS_KEYCTX_RX_SEQCTR_V(3) | \
+ TLS_KEYCTX_RXAUTH_MODE_V(4) | \
+ TLS_KEYCTX_RXCIPH_MODE_V(2) | \
+ TLS_KEYCTX_RXFLIT_CNT_V((ctx_len)))
+
#define FILL_WR_OP_CCTX_SIZE \
htonl( \
FW_CRYPTO_LOOKASIDE_WR_OPCODE_V( \
diff --git a/drivers/crypto/chelsio/chcr_core.h
b/drivers/crypto/chelsio/chcr_core.h
index 3c29ee0..77056a9 100644
--- a/drivers/crypto/chelsio/chcr_core.h
+++ b/drivers/crypto/chelsio/chcr_core.h
@@ -65,10 +65,58 @@
struct _key_ctx {
__be32 ctx_hdr;
u8 salt[MAX_SALT];
- __be64 reserverd;
+ __be64 iv_to_auth;
unsigned char key[0];
};
+#define KEYCTX_TX_WR_IV_S 55
+#define KEYCTX_TX_WR_IV_M 0x1ffULL
+#define KEYCTX_TX_WR_IV_V(x) ((x) << KEYCTX_TX_WR_IV_S)
+#define KEYCTX_TX_WR_IV_G(x) \
+ (((x) >> KEYCTX_TX_WR_IV_S) & KEYCTX_TX_WR_IV_M)
+
+#define KEYCTX_TX_WR_AAD_S 47
+#define KEYCTX_TX_WR_AAD_M 0xffULL
+#define KEYCTX_TX_WR_AAD_V(x) ((x) << KEYCTX_TX_WR_AAD_S)
+#define KEYCTX_TX_WR_AAD_G(x) (((x) >> KEYCTX_TX_WR_AAD_S) & \
+ KEYCTX_TX_WR_AAD_M)
+
+#define KEYCTX_TX_WR_AADST_S 39
+#define KEYCTX_TX_WR_AADST_M 0xffULL
+#define KEYCTX_TX_WR_AADST_V(x) ((x) << KEYCTX_TX_WR_AADST_S)
+#define KEYCTX_TX_WR_AADST_G(x) \
+ (((x) >> KEYCTX_TX_WR_AADST_S) & KEYCTX_TX_WR_AADST_M)
+
+#define KEYCTX_TX_WR_CIPHER_S 30
+#define KEYCTX_TX_WR_CIPHER_M 0x1ffULL
+#define KEYCTX_TX_WR_CIPHER_V(x) ((x) << KEYCTX_TX_WR_CIPHER_S)
+#define KEYCTX_TX_WR_CIPHER_G(x) \
+ (((x) >> KEYCTX_TX_WR_CIPHER_S) & KEYCTX_TX_WR_CIPHER_M)
+
+#define KEYCTX_TX_WR_CIPHERST_S 23
+#define KEYCTX_TX_WR_CIPHERST_M 0x7f
+#define KEYCTX_TX_WR_CIPHERST_V(x) ((x) << KEYCTX_TX_WR_CIPHERST_S)
+#define KEYCTX_TX_WR_CIPHERST_G(x) \
+ (((x) >> KEYCTX_TX_WR_CIPHERST_S) & KEYCTX_TX_WR_CIPHERST_M)
+
+#define KEYCTX_TX_WR_AUTH_S 14
+#define KEYCTX_TX_WR_AUTH_M 0x1ff
+#define KEYCTX_TX_WR_AUTH_V(x) ((x) << KEYCTX_TX_WR_AUTH_S)
+#define KEYCTX_TX_WR_AUTH_G(x) \
+ (((x) >> KEYCTX_TX_WR_AUTH_S) & KEYCTX_TX_WR_AUTH_M)
+
+#define KEYCTX_TX_WR_AUTHST_S 7
+#define KEYCTX_TX_WR_AUTHST_M 0x7f
+#define KEYCTX_TX_WR_AUTHST_V(x) ((x) << KEYCTX_TX_WR_AUTHST_S)
+#define KEYCTX_TX_WR_AUTHST_G(x) \
+ (((x) >> KEYCTX_TX_WR_AUTHST_S) & KEYCTX_TX_WR_AUTHST_M)
+
+#define KEYCTX_TX_WR_AUTHIN_S 0
+#define KEYCTX_TX_WR_AUTHIN_M 0x7f
+#define KEYCTX_TX_WR_AUTHIN_V(x) ((x) << KEYCTX_TX_WR_AUTHIN_S)
+#define KEYCTX_TX_WR_AUTHIN_G(x) \
+ (((x) >> KEYCTX_TX_WR_AUTHIN_S) & KEYCTX_TX_WR_AUTH
[Crypto v5 10/12] chtls: Inline crypto request Tx/Rx
TLS handler for record transmit and receive.
Create Inline TLS work request and post to FW.
Signed-off-by: Atul Gupta
---
drivers/crypto/chelsio/chtls/chtls_io.c | 1867 +++
1 file changed, 1867 insertions(+)
create mode 100644 drivers/crypto/chelsio/chtls/chtls_io.c
diff --git a/drivers/crypto/chelsio/chtls/chtls_io.c
b/drivers/crypto/chelsio/chtls/chtls_io.c
new file mode 100644
index 000..a0f03fb
--- /dev/null
+++ b/drivers/crypto/chelsio/chtls/chtls_io.c
@@ -0,0 +1,1867 @@
+/*
+ * Copyright (c) 2017 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Written by: Atul Gupta (atul.gu...@chelsio.com)
+ */
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "chtls.h"
+#include "chtls_cm.h"
+
+static bool is_tls_hw(struct chtls_sock *csk)
+{
+ return csk->tlshws.ofld;
+}
+
+static bool is_tls_rx(struct chtls_sock *csk)
+{
+ return (csk->tlshws.rxkey >= 0);
+}
+
+static bool is_tls_tx(struct chtls_sock *csk)
+{
+ return (csk->tlshws.txkey >= 0);
+}
+
+static bool is_tls_skb(struct chtls_sock *csk, const struct sk_buff *skb)
+{
+ return (is_tls_hw(csk) && skb_ulp_tls_skb_flags(skb));
+}
+
+static int key_size(void *sk)
+{
+ return 16; /* Key on DDR */
+}
+
+#define ceil(x, y) \
+ ({ unsigned long __x = (x), __y = (y); (__x + __y - 1) / __y; })
+
+static int data_sgl_len(const struct sk_buff *skb)
+{
+ unsigned int cnt;
+
+ cnt = skb_shinfo(skb)->nr_frags;
+ return (sgl_len(cnt) * 8);
+}
+
+static int nos_ivs(struct sock *sk, unsigned int size)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+
+ return ceil(size, csk->tlshws.mfs);
+}
+
+#define TLS_WR_CPL_LEN \
+ (sizeof(struct fw_tlstx_data_wr) + \
+ sizeof(struct cpl_tx_tls_sfo))
+
+static int is_ivs_imm(struct sock *sk, const struct sk_buff *skb)
+{
+ int ivs_size = nos_ivs(sk, skb->len) * CIPHER_BLOCK_SIZE;
+ int hlen = TLS_WR_CPL_LEN + data_sgl_len(skb);
+
+ if ((hlen + key_size(sk) + ivs_size) <
+ MAX_IMM_OFLD_TX_DATA_WR_LEN) {
+ ULP_SKB_CB(skb)->ulp.tls.iv = 1;
+ return 1;
+ }
+ ULP_SKB_CB(skb)->ulp.tls.iv = 0;
+ return 0;
+}
+
+static int max_ivs_size(struct sock *sk, int size)
+{
+ return (nos_ivs(sk, size) * CIPHER_BLOCK_SIZE);
+}
+
+static int ivs_size(struct sock *sk, const struct sk_buff *skb)
+{
+ return (is_ivs_imm(sk, skb) ? (nos_ivs(sk, skb->len) *
+CIPHER_BLOCK_SIZE) : 0);
+}
+
+static int flowc_wr_credits(int nparams, int *flowclenp)
+{
+ int flowclen16, flowclen;
+
+ flowclen = offsetof(struct fw_flowc_wr, mnemval[nparams]);
+ flowclen16 = DIV_ROUND_UP(flowclen, 16);
+ flowclen = flowclen16 * 16;
+
+ if (flowclenp)
+ *flowclenp = flowclen;
+
+ return flowclen16;
+}
+
+static struct sk_buff *create_flowc_wr_skb(struct sock *sk,
+ struct fw_flowc_wr *flowc,
+ int flowclen)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+ struct sk_buff *skb;
+
+ skb = alloc_skb(flowclen, GFP_ATOMIC);
+ if (!skb)
+ return NULL;
+
+ memcpy(__skb_put(skb, flowclen), flowc, flowclen);
+ set_queue(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA, sk);
+
+ return skb;
+}
+
+static int send_flowc_wr(struct sock *sk, struct fw_flowc_wr *flowc,
+int flowclen)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+ bool syn_sent = (sk->sk_state == TCP_SYN_SENT);
+ struct tcp_sock *tp = tcp_sk(sk);
+ int flowclen16 = flowclen / 16;
+ struct sk_buff *skb;
+
+ if (csk_flag(sk, CSK_TX_DATA_SENT)) {
+ skb = create_flowc_wr_skb(sk, flowc, flowclen);
+ if (!skb)
+ return -ENOMEM;
+
+ if (syn_sent)
+ __skb_queue_tail(&csk->ooo_queue, skb);
+ else
+ skb_entail(sk, skb,
+ ULPCB_FLAG_NO_HDR | ULPCB_FLAG_NO_APPEND);
+ return 0;
+ }
+
+ if (!syn_sent) {
+ int ret;
+
+ ret = cxgb4_immdata_send(csk->egress_dev,
+csk->txq_idx,
+flowc, flowclen);
+ if (!ret)
+ return flowclen16;
+ }
+ skb = create_flowc_wr_skb(sk, flowc, flowclen);
+ if (!skb)
+ return -
[Crypto v5 05/12] cxgb4: Inline TLS FW Interface
Key area size in hw-config file. CPL struct for TLS request
and response. Work request for Inline TLS.
Signed-off-by: Atul Gupta
---
drivers/net/ethernet/chelsio/cxgb4/t4_msg.h | 121 ++-
drivers/net/ethernet/chelsio/cxgb4/t4_regs.h | 2 +
drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h | 165 +-
3 files changed, 283 insertions(+), 5 deletions(-)
diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h
b/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h
index 7e12f24..9a56e0d 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h
@@ -81,6 +81,7 @@ enum {
CPL_RX_ISCSI_CMP = 0x45,
CPL_TRACE_PKT_T5 = 0x48,
CPL_RX_ISCSI_DDP = 0x49,
+ CPL_RX_TLS_CMP= 0x4E,
CPL_RDMA_READ_REQ = 0x60,
@@ -88,6 +89,7 @@ enum {
CPL_ACT_OPEN_REQ6 = 0x83,
CPL_TX_TLS_PDU =0x88,
+ CPL_TX_TLS_SFO= 0x89,
CPL_TX_SEC_PDU= 0x8A,
CPL_TX_TLS_ACK= 0x8B,
@@ -97,6 +99,7 @@ enum {
CPL_RX_MPS_PKT= 0xAF,
CPL_TRACE_PKT = 0xB0,
+ CPL_TLS_DATA = 0xB1,
CPL_ISCSI_DATA= 0xB2,
CPL_FW4_MSG = 0xC0,
@@ -150,6 +153,7 @@ enum {
ULP_MODE_RDMA = 4,
ULP_MODE_TCPDDP= 5,
ULP_MODE_FCOE = 6,
+ ULP_MODE_TLS = 8,
};
enum {
@@ -1414,6 +1418,14 @@ struct cpl_tx_data {
#define TX_FORCE_S 13
#define TX_FORCE_V(x) ((x) << TX_FORCE_S)
+#define TX_SHOVE_S14
+#define TX_SHOVE_V(x) ((x) << TX_SHOVE_S)
+
+#define TX_ULP_MODE_S10
+#define TX_ULP_MODE_M0x7
+#define TX_ULP_MODE_V(x) ((x) << TX_ULP_MODE_S)
+#define TX_ULP_MODE_G(x) (((x) >> TX_ULP_MODE_S) & TX_ULP_MODE_M)
+
#define T6_TX_FORCE_S 20
#define T6_TX_FORCE_V(x) ((x) << T6_TX_FORCE_S)
#define T6_TX_FORCE_F T6_TX_FORCE_V(1U)
@@ -1428,12 +1440,21 @@ enum {
ULP_TX_SC_NOOP = 0x80,
ULP_TX_SC_IMM = 0x81,
ULP_TX_SC_DSGL = 0x82,
- ULP_TX_SC_ISGL = 0x83
+ ULP_TX_SC_ISGL = 0x83,
+ ULP_TX_SC_MEMRD = 0x86
};
#define ULPTX_CMD_S24
#define ULPTX_CMD_V(x) ((x) << ULPTX_CMD_S)
+#define ULPTX_LEN16_S0
+#define ULPTX_LEN16_M0xFF
+#define ULPTX_LEN16_V(x) ((x) << ULPTX_LEN16_S)
+
+#define ULP_TX_SC_MORE_S 23
+#define ULP_TX_SC_MORE_V(x) ((x) << ULP_TX_SC_MORE_S)
+#define ULP_TX_SC_MORE_F ULP_TX_SC_MORE_V(1U)
+
struct ulptx_sge_pair {
__be32 len[2];
__be64 addr[2];
@@ -1948,4 +1969,102 @@ enum {
X_CPL_RX_MPS_PKT_TYPE_QFC = 1 << 2,
X_CPL_RX_MPS_PKT_TYPE_PTP = 1 << 3
};
+
+struct cpl_tx_tls_sfo {
+ __be32 op_to_seg_len;
+ __be32 pld_len;
+ __be32 type_protover;
+ __be32 r1_lo;
+ __be32 seqno_numivs;
+ __be32 ivgen_hdrlen;
+ __be64 scmd1;
+};
+
+/* cpl_tx_tls_sfo macros */
+#define CPL_TX_TLS_SFO_OPCODE_S 24
+#define CPL_TX_TLS_SFO_OPCODE_V(x) ((x) << CPL_TX_TLS_SFO_OPCODE_S)
+
+#define CPL_TX_TLS_SFO_DATA_TYPE_S 20
+#define CPL_TX_TLS_SFO_DATA_TYPE_V(x) ((x) << CPL_TX_TLS_SFO_DATA_TYPE_S)
+
+#define CPL_TX_TLS_SFO_CPL_LEN_S16
+#define CPL_TX_TLS_SFO_CPL_LEN_V(x) ((x) << CPL_TX_TLS_SFO_CPL_LEN_S)
+
+#define CPL_TX_TLS_SFO_SEG_LEN_S0
+#define CPL_TX_TLS_SFO_SEG_LEN_M0x
+#define CPL_TX_TLS_SFO_SEG_LEN_V(x) ((x) << CPL_TX_TLS_SFO_SEG_LEN_S)
+#define CPL_TX_TLS_SFO_SEG_LEN_G(x) \
+ (((x) >> CPL_TX_TLS_SFO_SEG_LEN_S) & CPL_TX_TLS_SFO_SEG_LEN_M)
+
+#define CPL_TX_TLS_SFO_TYPE_S 24
+#define CPL_TX_TLS_SFO_TYPE_M 0xff
+#define CPL_TX_TLS_SFO_TYPE_V(x)((x) << CPL_TX_TLS_SFO_TYPE_S)
+#define CPL_TX_TLS_SFO_TYPE_G(x)\
+ (((x) >> CPL_TX_TLS_SFO_TYPE_S) & CPL_TX_TLS_SFO_TYPE_M)
+
+#define CPL_TX_TLS_SFO_PROTOVER_S 8
+#define CPL_TX_TLS_SFO_PROTOVER_M 0x
+#define CPL_TX_TLS_SFO_PROTOVER_V(x)((x) << CPL_TX_TLS_SFO_PROTOVER_S)
+#define CPL_TX_TLS_SFO_PROTOVER_G(x)\
+ (((x) >> CPL_TX_TLS_SFO_PROTOVER_S) & CPL_TX_TLS_SFO_PROTOVER_M)
+
+struct cpl_tls_data {
+ struct rss_header rsshdr;
+ union opcode_tid ot;
+ __be32 length_pkd;
+ __be32 seq;
+ __be32 r1;
+};
+
+#define CPL_TLS_DATA_OPCODE_S 24
+#define CPL_TLS_DATA_OPCODE_M 0xff
+#define CPL_TLS_DATA_OPCODE_V(x)((x) << CPL_TLS_DATA_OPCODE_S)
+#define CPL_TLS_DATA_OPCODE_G(x)\
+ (((x) >> CPL_TLS_DATA_OPCODE_S) & CPL_TLS_DATA_OPCODE_M)
+
+#define CPL_TLS_DATA_TID_S 0
+#define CPL_TLS_DATA_TID_M 0xff
+#define CPL_TLS_DATA_TID_V(x) ((x) << CPL_TLS_DATA_TID_S)
+#define CPL_TLS_DATA_TID_G(x) \
+ (((x) >> CPL_TLS_DATA_TID_S) &
[Crypto v5 12/12] Makefile Kconfig
Entry for Inline TLS as another driver dependent on cxgb4 and chcr Signed-off-by: Atul Gupta --- drivers/crypto/chelsio/Kconfig| 11 +++ drivers/crypto/chelsio/Makefile | 1 + drivers/crypto/chelsio/chtls/Makefile | 4 3 files changed, 16 insertions(+) create mode 100644 drivers/crypto/chelsio/chtls/Makefile diff --git a/drivers/crypto/chelsio/Kconfig b/drivers/crypto/chelsio/Kconfig index 5ae9f87..930d82d 100644 --- a/drivers/crypto/chelsio/Kconfig +++ b/drivers/crypto/chelsio/Kconfig @@ -29,3 +29,14 @@ config CHELSIO_IPSEC_INLINE default n ---help--- Enable support for IPSec Tx Inline. + +config CRYPTO_DEV_CHELSIO_TLS +tristate "Chelsio Crypto Inline TLS Driver" +depends on CHELSIO_T4 +depends on TLS +select CRYPTO_DEV_CHELSIO +---help--- + Support Chelsio Inline TLS with Chelsio crypto accelerator. + + To compile this driver as a module, choose M here: the module + will be called chtls. diff --git a/drivers/crypto/chelsio/Makefile b/drivers/crypto/chelsio/Makefile index eaecaf1..639e571 100644 --- a/drivers/crypto/chelsio/Makefile +++ b/drivers/crypto/chelsio/Makefile @@ -3,3 +3,4 @@ ccflags-y := -Idrivers/net/ethernet/chelsio/cxgb4 obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chcr.o chcr-objs := chcr_core.o chcr_algo.o chcr-$(CONFIG_CHELSIO_IPSEC_INLINE) += chcr_ipsec.o +obj-$(CONFIG_CRYPTO_DEV_CHELSIO_TLS) += chtls/ diff --git a/drivers/crypto/chelsio/chtls/Makefile b/drivers/crypto/chelsio/chtls/Makefile new file mode 100644 index 000..df13795 --- /dev/null +++ b/drivers/crypto/chelsio/chtls/Makefile @@ -0,0 +1,4 @@ +ccflags-y := -Idrivers/net/ethernet/chelsio/cxgb4 -Idrivers/crypto/chelsio/ + +obj-$(CONFIG_CRYPTO_DEV_CHELSIO_TLS) += chtls.o +chtls-objs := chtls_main.o chtls_cm.o chtls_io.o chtls_hw.o -- 1.8.3.1
[Crypto v5 08/12] chtls: Key program
Program the tx and rx key on chip.
Signed-off-by: Atul Gupta
---
drivers/crypto/chelsio/chtls/chtls_hw.c | 394
1 file changed, 394 insertions(+)
create mode 100644 drivers/crypto/chelsio/chtls/chtls_hw.c
diff --git a/drivers/crypto/chelsio/chtls/chtls_hw.c
b/drivers/crypto/chelsio/chtls/chtls_hw.c
new file mode 100644
index 000..c3e17159
--- /dev/null
+++ b/drivers/crypto/chelsio/chtls/chtls_hw.c
@@ -0,0 +1,394 @@
+/*
+ * Copyright (c) 2017 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Written by: Atul Gupta (atul.gu...@chelsio.com)
+ */
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "chtls.h"
+#include "chtls_cm.h"
+
+static void __set_tcb_field_direct(struct chtls_sock *csk,
+ struct cpl_set_tcb_field *req, u16 word,
+ u64 mask, u64 val, u8 cookie, int no_reply)
+{
+ struct ulptx_idata *sc;
+
+ INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, csk->tid);
+ req->wr.wr_mid |= htonl(FW_WR_FLOWID_V(csk->tid));
+ req->reply_ctrl = htons(NO_REPLY_V(no_reply) |
+ QUEUENO_V(csk->rss_qid));
+ req->word_cookie = htons(TCB_WORD_V(word) | TCB_COOKIE_V(cookie));
+ req->mask = cpu_to_be64(mask);
+ req->val = cpu_to_be64(val);
+ sc = (struct ulptx_idata *)(req + 1);
+ sc->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP));
+ sc->len = htonl(0);
+}
+
+static void __set_tcb_field(struct sock *sk, struct sk_buff *skb, u16 word,
+ u64 mask, u64 val, u8 cookie, int no_reply)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+ struct cpl_set_tcb_field *req;
+ struct ulptx_idata *sc;
+ unsigned int wrlen = roundup(sizeof(*req) + sizeof(*sc), 16);
+
+ req = (struct cpl_set_tcb_field *)__skb_put(skb, wrlen);
+ __set_tcb_field_direct(csk, req, word, mask, val, cookie, no_reply);
+ set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id);
+}
+
+static int chtls_set_tcb_field(struct sock *sk, u16 word, u64 mask, u64 val)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+ struct sk_buff *skb;
+ struct cpl_set_tcb_field *req;
+ struct ulptx_idata *sc;
+ unsigned int wrlen = roundup(sizeof(*req) + sizeof(*sc), 16);
+ unsigned int credits_needed = DIV_ROUND_UP(wrlen, 16);
+
+ skb = alloc_skb(wrlen, GFP_ATOMIC);
+ if (!skb)
+ return -ENOMEM;
+
+ __set_tcb_field(sk, skb, word, mask, val, 0, 1);
+ set_queue(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA, sk);
+ csk->wr_credits -= credits_needed;
+ csk->wr_unacked += credits_needed;
+ enqueue_wr(csk, skb);
+ cxgb4_ofld_send(csk->egress_dev, skb);
+ return 0;
+}
+
+/*
+ * Set one of the t_flags bits in the TCB.
+ */
+int chtls_set_tcb_tflag(struct sock *sk, unsigned int bit_pos, int val)
+{
+ return chtls_set_tcb_field(sk, 1, 1ULL << bit_pos,
+ val << bit_pos);
+}
+
+static int chtls_set_tcb_keyid(struct sock *sk, int keyid)
+{
+ return chtls_set_tcb_field(sk, 31, 0xULL, keyid);
+}
+
+static int chtls_set_tcb_seqno(struct sock *sk)
+{
+ return chtls_set_tcb_field(sk, 28, ~0ULL, 0);
+}
+
+static int chtls_set_tcb_quiesce(struct sock *sk, int val)
+{
+ return chtls_set_tcb_field(sk, 1, (1ULL << TF_RX_QUIESCE_S),
+ TF_RX_QUIESCE_V(val));
+}
+
+static void *chtls_alloc_mem(unsigned long size)
+{
+ void *p = kmalloc(size, GFP_KERNEL);
+
+ if (!p)
+ p = vmalloc(size);
+ if (p)
+ memset(p, 0, size);
+ return p;
+}
+
+static void chtls_free_mem(void *addr)
+{
+ unsigned long p = (unsigned long)addr;
+
+ if (p >= VMALLOC_START && p < VMALLOC_END)
+ vfree(addr);
+ else
+ kfree(addr);
+}
+
+/* TLS Key bitmap processing */
+int chtls_init_kmap(struct chtls_dev *cdev, struct cxgb4_lld_info *lldi)
+{
+ unsigned int num_key_ctx, bsize;
+
+ num_key_ctx = (lldi->vr->key.size / TLS_KEY_CONTEXT_SZ);
+ bsize = BITS_TO_LONGS(num_key_ctx);
+
+ cdev->kmap.size = num_key_ctx;
+ cdev->kmap.available = bsize;
+ cdev->kmap.addr = chtls_alloc_mem(sizeof(*cdev->kmap.addr) *
+ bsize);
+ if (!cdev->kmap.addr)
+ return -1;
+
+ cdev->kmap.start = lldi->vr->key.start;
+ spin_lock_init(&cdev->kmap.lock);
+ return 0;
+}
+
+void chtls_free_kmap(st
RE: [Crypto v5 03/12] support for inline tls
-Original Message-
From: Dave Watson [mailto:davejwat...@fb.com]
Sent: Thursday, February 15, 2018 9:22 PM
To: Atul Gupta
Cc: da...@davemloft.net; herb...@gondor.apana.org.au; s...@queasysnail.net;
linux-crypto@vger.kernel.org; net...@vger.kernel.org; Ganesh GR
Subject: Re: [Crypto v5 03/12] support for inline tls
On 02/15/18 12:24 PM, Atul Gupta wrote:
> @@ -401,6 +430,15 @@ static int do_tls_setsockopt_tx(struct sock *sk, char
> __user *optval,
> goto out;
> }
>
> + rc = get_tls_offload_dev(sk);
> + if (rc) {
> + goto out;
> + } else {
> + /* Retain HW unhash for cleanup and move to SW Tx */
> + sk->sk_prot[TLS_BASE_TX].unhash =
> + sk->sk_prot[TLS_FULL_HW].unhash;
Isn't sk->sk_prot a pointer to a global shared struct here still? It looks
like this would actually modify the global struct, and not just for this sk.
Yes, its global. It require add on check to modify only when tls_offload dev
list has an entry. I will revisit and correct. Can you look through other
changes please?
Thanks
Atul
RE: [Crypto v5 03/12] support for inline tls
> > > @@ -401,6 +430,15 @@ static int do_tls_setsockopt_tx(struct sock *sk,
> > > char __user *optval,
> > > goto out;
> > > }
> > >
> > > + rc = get_tls_offload_dev(sk);
> > > + if (rc) {
> > > + goto out;
> > > + } else {
> > > + /* Retain HW unhash for cleanup and move to SW Tx */
> > > + sk->sk_prot[TLS_BASE_TX].unhash =
> > > + sk->sk_prot[TLS_FULL_HW].unhash;
> >
> > Isn't sk->sk_prot a pointer to a global shared struct here still? It looks
> > like this would actually modify the global struct, and not just for this sk.
> Yes, its global. It require add on check to modify only when tls_offload dev
> list has an entry. I will revisit and correct.
>
> Can you look through other changes please?
I looked through 1,2,3,11 (the tls-related ones) and don't have any other code
comments. Patch 11 commit message still mentions ULP, could use updating /
clarification.
Thank you, I will collate and clean for v6.
[Crypto v6 00/12] Chelsio Inline TLS
Series for Chelsio Inline TLS driver (chtls.ko)
Use tls ULP infrastructure to register chtls as Inline TLS driver.
Chtls use TCP Sockets to transmit and receive TLS record. TCP proto_ops is
extended to offload TLS record.
T6 adapter provides the following features:
-TLS record offload, TLS header, encrypt, digest and transmit
-TLS record receive and decrypt
-TLS keys store
-TCP/IP engine
-TLS engine
-GCM crypto engine [support CBC also]
TLS provides security at the transport layer. It uses TCP to provide reliable
end-to-end transport of application data. It relies on TCP for any
retransmission. TLS session comprises of three parts:
a. TCP/IP connection
b. TLS handshake
c. Record layer processing
TLS handshake state machine is executed in host (refer standard implementation
eg. OpenSSL). Setsockopt [SOL_TCP, TCP_ULP] initialize TCP proto-ops for
Chelsio inline tls support. setsockopt(sock, SOL_TCP, TCP_ULP, "tls",
sizeof("tls"));
Tx and Rx Keys are decided during handshake and programmed onto the chip after
CCS is exchanged.
struct tls12_crypto_info_aes_gcm_128 crypto_info setsockopt(sock, SOL_TLS,
TLS_TX, &crypto_info, sizeof(crypto_info)) Finish is the first
encrypted/decrypted message tx/rx inline.
On the Tx path TLS engine receive plain text from openssl, insert IV, fetches
the tx key, create cipher text records and generate MAC. TLS header is added to
cipher text and forward to TCP/IP engine for transport layer processing and
transmission on wire.
TX:
Application--openssl--chtls---TLS engine---encrypt/auth---TCP/IP engine---wire.
On the Rx side, data received is PDU aligned at record boundaries. TLS
processes only the complete record. If rx key is programmed on CCS receive,
data is decrypted and plain text is posted to host.
RX:
Wire--cipher-text--TCP/IP engine [PDU align]---TLS engine---
decrypt/auth---plain-text--chtls--openssl--application
v6: modify prot only for FULL_HW
-corrected commit message for patch 11
v5: set TLS_FULL_HW for registered inline tls drivers
-set TLS_FULL_HW prot for offload connection else move
to TLS_SW_TX
-Case handled for interface with same IP [Dave Miller]
-Removed Specific IP and INADDR_ANY handling [v4]
v4: removed chtls ULP type, retained tls ULP
-registered chtls with net tls
-defined struct tls_device to register the Inline drivers
-ethtool interface tls-inline to enable Inline TLS for interface
-prot update to support inline TLS
v3: fixed the kbuild test issues
-made few funtions static
-initialized few variables
v2: fixed the following based on the review comments of Stephan Mueller,
Stefano Brivio and Hannes Frederic
-Added more details in cover letter
-Fixed indentation and formating issues
-Using aes instead of aes-generic
-memset key info after programing the key on chip
-reordered the patch sequence
Atul Gupta (12):
tls: tls_device struct to register TLS drivers
ethtool: enable Inline TLS in HW
tls: support for inline tls
chtls: structure and macro definiton
cxgb4: Inline TLS FW Interface
cxgb4: LLD driver changes to enable TLS
chcr: Key Macro
chtls: Key program
chtls: CPL handler definition
chtls: Inline crypto request Tx/Rx
chtls: Register chtls Inline TLS with net tls
Makefile Kconfig
drivers/crypto/chelsio/Kconfig | 11 +
drivers/crypto/chelsio/Makefile |1 +
drivers/crypto/chelsio/chcr_algo.h | 42 +
drivers/crypto/chelsio/chcr_core.h | 55 +-
drivers/crypto/chelsio/chtls/Makefile |4 +
drivers/crypto/chelsio/chtls/chtls.h| 487 ++
drivers/crypto/chelsio/chtls/chtls_cm.c | 2041 +++
drivers/crypto/chelsio/chtls/chtls_cm.h | 202 +++
drivers/crypto/chelsio/chtls/chtls_hw.c | 394 +
drivers/crypto/chelsio/chtls/chtls_io.c | 1867 +
drivers/crypto/chelsio/chtls/chtls_main.c | 600 +++
drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c | 32 +-
drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h |7 +
drivers/net/ethernet/chelsio/cxgb4/sge.c| 98 +-
drivers/net/ethernet/chelsio/cxgb4/t4_msg.h | 121 +-
drivers/net/ethernet/chelsio/cxgb4/t4_regs.h|2 +
drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h | 165 +-
include/linux/netdev_features.h |2 +
include/net/tls.h | 23 +
include/uapi/linux/tls.h|1 +
net/core/ethtool.c |1 +
net/ipv4/tcp_minisocks.c|1 +
net/tls/tls_main.c | 123 +-
23 files changed, 6256 insertions(+), 24 deletions(-)
create mode 100644 drivers/crypto/chelsio/chtls/Makefile
create mode 100644 drivers/crypto/chelsio/chtls/chtls.h
create mode 100644 drivers/crypto/chelsio/chtls
[Crypto v6 01/12] tls: tls_device struct to register TLS drivers
tls_device structure to register Inline TLS drivers
with net/tls
Signed-off-by: Atul Gupta
---
include/net/tls.h | 23 +++
1 file changed, 23 insertions(+)
diff --git a/include/net/tls.h b/include/net/tls.h
index 4913430..e315bf9 100644
--- a/include/net/tls.h
+++ b/include/net/tls.h
@@ -55,6 +55,27 @@
#define TLS_RECORD_TYPE_DATA 0x17
#define TLS_AAD_SPACE_SIZE 13
+#define TLS_DEVICE_NAME_MAX32
+
+enum {
+ TLS_BASE_TX,
+ TLS_SW_TX,
+ TLS_FULL_HW, /* TLS record processed Inline */
+ TLS_NUM_CONFIG,
+};
+extern struct proto tls_prots[TLS_NUM_CONFIG];
+
+struct tls_device {
+ char name[TLS_DEVICE_NAME_MAX];
+ struct list_head dev_list;
+
+ /* netdev present in registered inline tls driver */
+ int (*netdev)(struct tls_device *device,
+ struct net_device *netdev);
+ int (*feature)(struct tls_device *device);
+ int (*hash)(struct tls_device *device, struct sock *sk);
+ void (*unhash)(struct tls_device *device, struct sock *sk);
+};
struct tls_sw_context {
struct crypto_aead *aead_send;
@@ -256,5 +277,7 @@ static inline struct tls_offload_context *tls_offload_ctx(
int tls_proccess_cmsg(struct sock *sk, struct msghdr *msg,
unsigned char *record_type);
+void tls_register_device(struct tls_device *device);
+void tls_unregister_device(struct tls_device *device);
#endif /* _TLS_OFFLOAD_H */
--
1.8.3.1
[Crypto v6 02/12] ethtool: enable Inline TLS in HW
Signed-off-by: Atul Gupta
---
include/linux/netdev_features.h | 2 ++
net/core/ethtool.c | 1 +
2 files changed, 3 insertions(+)
diff --git a/include/linux/netdev_features.h b/include/linux/netdev_features.h
index db84c51..aacabe2 100644
--- a/include/linux/netdev_features.h
+++ b/include/linux/netdev_features.h
@@ -79,6 +79,7 @@ enum {
NETIF_F_RX_UDP_TUNNEL_PORT_BIT, /* Offload of RX port for UDP tunnels */
NETIF_F_GRO_HW_BIT, /* Hardware Generic receive offload */
+ NETIF_F_HW_TLS_INLINE_BIT, /* Offload TLS record */
/*
* Add your fresh new feature above and remember to update
@@ -145,6 +146,7 @@ enum {
#define NETIF_F_HW_ESP __NETIF_F(HW_ESP)
#define NETIF_F_HW_ESP_TX_CSUM __NETIF_F(HW_ESP_TX_CSUM)
#defineNETIF_F_RX_UDP_TUNNEL_PORT __NETIF_F(RX_UDP_TUNNEL_PORT)
+#define NETIF_F_HW_TLS_INLINE __NETIF_F(HW_TLS_INLINE)
#define for_each_netdev_feature(mask_addr, bit)\
for_each_set_bit(bit, (unsigned long *)mask_addr, NETDEV_FEATURE_COUNT)
diff --git a/net/core/ethtool.c b/net/core/ethtool.c
index 494e6a5..ab16781 100644
--- a/net/core/ethtool.c
+++ b/net/core/ethtool.c
@@ -107,6 +107,7 @@ int ethtool_op_get_ts_info(struct net_device *dev, struct
ethtool_ts_info *info)
[NETIF_F_HW_ESP_BIT] = "esp-hw-offload",
[NETIF_F_HW_ESP_TX_CSUM_BIT] = "esp-tx-csum-hw-offload",
[NETIF_F_RX_UDP_TUNNEL_PORT_BIT] = "rx-udp_tunnel-port-offload",
+ [NETIF_F_HW_TLS_INLINE_BIT] = "tls-inline",
};
static const char
--
1.8.3.1
[Crypto v6 06/12] cxgb4: LLD driver changes to enable TLS
Read FW capability. Read key area size. Dump the TLS record count.
Signed-off-by: Atul Gupta
---
drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c | 32 +---
drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h | 7 ++
drivers/net/ethernet/chelsio/cxgb4/sge.c| 98 -
3 files changed, 126 insertions(+), 11 deletions(-)
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
index 56bc626..ab5937e 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
@@ -4284,18 +4284,32 @@ static int adap_init0(struct adapter *adap)
adap->num_ofld_uld += 2;
}
if (caps_cmd.cryptocaps) {
- /* Should query params here...TODO */
- params[0] = FW_PARAM_PFVF(NCRYPTO_LOOKASIDE);
- ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2,
- params, val);
- if (ret < 0) {
- if (ret != -EINVAL)
+ if (ntohs(caps_cmd.cryptocaps) &
+ FW_CAPS_CONFIG_CRYPTO_LOOKASIDE) {
+ params[0] = FW_PARAM_PFVF(NCRYPTO_LOOKASIDE);
+ ret = t4_query_params(adap, adap->mbox, adap->pf, 0,
+ 2, params, val);
+ if (ret < 0) {
+ if (ret != -EINVAL)
+ goto bye;
+ } else {
+ adap->vres.ncrypto_fc = val[0];
+ }
+ adap->num_ofld_uld += 1;
+ }
+ if (ntohs(caps_cmd.cryptocaps) &
+ FW_CAPS_CONFIG_TLS_INLINE) {
+ params[0] = FW_PARAM_PFVF(TLS_START);
+ params[1] = FW_PARAM_PFVF(TLS_END);
+ ret = t4_query_params(adap, adap->mbox, adap->pf, 0,
+ 2, params, val);
+ if (ret < 0)
goto bye;
- } else {
- adap->vres.ncrypto_fc = val[0];
+ adap->vres.key.start = val[0];
+ adap->vres.key.size = val[1] - val[0] + 1;
+ adap->num_uld += 1;
}
adap->params.crypto = ntohs(caps_cmd.cryptocaps);
- adap->num_uld += 1;
}
#undef FW_PARAM_PFVF
#undef FW_PARAM_DEV
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h
b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h
index a14e8db..3d3ef3f 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h
@@ -237,6 +237,7 @@ enum cxgb4_uld {
CXGB4_ULD_ISCSI,
CXGB4_ULD_ISCSIT,
CXGB4_ULD_CRYPTO,
+ CXGB4_ULD_TLS,
CXGB4_ULD_MAX
};
@@ -287,6 +288,7 @@ struct cxgb4_virt_res { /* virtualized
HW resources */
struct cxgb4_range qp;
struct cxgb4_range cq;
struct cxgb4_range ocq;
+ struct cxgb4_range key;
unsigned int ncrypto_fc;
};
@@ -298,6 +300,9 @@ struct chcr_stats_debug {
atomic_t error;
atomic_t fallback;
atomic_t ipsec_cnt;
+ atomic_t tls_pdu_tx;
+ atomic_t tls_pdu_rx;
+ atomic_t tls_key;
};
#define OCQ_WIN_OFFSET(pdev, vres) \
@@ -378,6 +383,8 @@ struct cxgb4_uld_info {
int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p);
int cxgb4_unregister_uld(enum cxgb4_uld type);
int cxgb4_ofld_send(struct net_device *dev, struct sk_buff *skb);
+int cxgb4_immdata_send(struct net_device *dev, unsigned int idx,
+ const void *src, unsigned int len);
int cxgb4_crypto_send(struct net_device *dev, struct sk_buff *skb);
unsigned int cxgb4_dbfifo_count(const struct net_device *dev, int lpfifo);
unsigned int cxgb4_port_chan(const struct net_device *dev);
diff --git a/drivers/net/ethernet/chelsio/cxgb4/sge.c
b/drivers/net/ethernet/chelsio/cxgb4/sge.c
index 6e310a0..32e3779 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/sge.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/sge.c
@@ -1740,9 +1740,9 @@ static void txq_stop_maperr(struct sge_uld_txq *q)
* Stops an offload Tx queue that has become full and modifies the packet
* being written to request a wakeup.
*/
-static void ofldtxq_stop(struct sge_uld_txq *q, struct sk_buff *skb)
+static void ofldtxq_stop(struct sge_uld_txq *q, void *src)
{
- struct fw_wr_hdr *wr = (struct fw_wr_hdr *)skb->data;
+ struct fw_wr_hdr *wr = (struct fw_wr_hdr *)src;
wr->lo |= htonl(FW_WR_EQUEQ_F | FW_WR_EQUIQ_F);
q->q.stops++;
@@ -2005,6 +2005,100 @@ int cxgb4_ofld_send(struct net_device *dev, struct
sk_buff *skb)
}
EXPORT_SY
[Crypto v6 04/12] chtls: structure and macro definiton
Inline TLS state, connection management. Supporting macros definition.
Signed-off-by: Atul Gupta
---
drivers/crypto/chelsio/chtls/chtls.h| 487
drivers/crypto/chelsio/chtls/chtls_cm.h | 202 +
2 files changed, 689 insertions(+)
create mode 100644 drivers/crypto/chelsio/chtls/chtls.h
create mode 100644 drivers/crypto/chelsio/chtls/chtls_cm.h
diff --git a/drivers/crypto/chelsio/chtls/chtls.h
b/drivers/crypto/chelsio/chtls/chtls.h
new file mode 100644
index 000..3ae7145
--- /dev/null
+++ b/drivers/crypto/chelsio/chtls/chtls.h
@@ -0,0 +1,487 @@
+/*
+ * Copyright (c) 2016 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef __CHTLS_H__
+#define __CHTLS_H__
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "t4fw_api.h"
+#include "t4_msg.h"
+#include "cxgb4.h"
+#include "cxgb4_uld.h"
+#include "l2t.h"
+#include "chcr_algo.h"
+#include "chcr_core.h"
+#include "chcr_crypto.h"
+
+#define CIPHER_BLOCK_SIZE 16
+#define MAX_IVS_PAGE256
+#define TLS_KEY_CONTEXT_SZ 64
+#define TLS_HEADER_LENGTH 5
+#define SCMD_CIPH_MODE_AES_GCM 2
+#define GCM_TAG_SIZE16
+#define AEAD_EXPLICIT_DATA_SIZE 8
+/* Any MFS size should work and come from openssl */
+#define TLS_MFS16384
+
+#define SOCK_INLINE (31)
+#define RSS_HDR sizeof(struct rss_header)
+
+enum {
+ CHTLS_KEY_CONTEXT_DSGL,
+ CHTLS_KEY_CONTEXT_IMM,
+ CHTLS_KEY_CONTEXT_DDR,
+};
+
+enum {
+ CHTLS_LISTEN_START,
+ CHTLS_LISTEN_STOP,
+};
+
+/* Flags for return value of CPL message handlers */
+enum {
+ CPL_RET_BUF_DONE = 1, /* buffer processing done */
+ CPL_RET_BAD_MSG = 2,/* bad CPL message */
+ CPL_RET_UNKNOWN_TID = 4 /* unexpected unknown TID */
+};
+
+#define TLS_RCV_ST_READ_HEADER 0xF0
+#define TLS_RCV_ST_READ_BODY0xF1
+#define TLS_RCV_ST_READ_DONE0xF2
+#define TLS_RCV_ST_READ_NB 0xF3
+
+#define RSPQ_HASH_BITS 5
+#define LISTEN_INFO_HASH_SIZE 32
+struct listen_info {
+ struct listen_info *next; /* Link to next entry */
+ struct sock *sk; /* The listening socket */
+ unsigned int stid; /* The server TID */
+};
+
+enum {
+ T4_LISTEN_START_PENDING,
+ T4_LISTEN_STARTED
+};
+
+enum csk_flags {
+ CSK_CALLBACKS_CHKD, /* socket callbacks have been sanitized */
+ CSK_ABORT_REQ_RCVD, /* received one ABORT_REQ_RSS message */
+ CSK_TX_MORE_DATA, /* sending ULP data; don't set SHOVE bit */
+ CSK_TX_WAIT_IDLE, /* suspend Tx until in-flight data is ACKed */
+ CSK_ABORT_SHUTDOWN, /* shouldn't send more abort requests */
+ CSK_ABORT_RPL_PENDING, /* expecting an abort reply */
+ CSK_CLOSE_CON_REQUESTED,/* we've sent a close_conn_req */
+ CSK_TX_DATA_SENT, /* sent a TX_DATA WR on this connection */
+ CSK_TX_FAILOVER,/* Tx traffic failing over */
+ CSK_UPDATE_RCV_WND, /* Need to update rcv window */
+ CSK_RST_ABORTED,/* outgoing RST was aborted */
+ CSK_TLS_HANDSHK,/* TLS Handshake */
+};
+
+struct listen_ctx {
+ struct sock *lsk;
+ struct chtls_dev *cdev;
+ u32 state;
+};
+
+struct key_map {
+ unsigned long *addr;
+ unsigned int start;
+ unsigned int available;
+ unsigned int size;
+ spinlock_t lock; /* lock for key id request from map */
+} __packed;
+
+struct tls_scmd {
+ u32 seqno_numivs;
+ u32 ivgen_hdrlen;
+};
+
+struct chtls_dev {
+ struct tls_device tlsdev;
+ struct list_head list;
+ struct cxgb4_lld_info *lldi;
+ struct pci_dev *pdev;
+ struct listen_info *listen_hash_tab[LISTEN_INFO_HASH_SIZE];
+ spinlock_t listen_lock; /* lock for listen list */
+ struct net_device **ports;
+ struct tid_info *tids;
+ unsigned int pfvf;
+ const unsigned short *mtus;
+
+ spinlock_t aidr_lock cacheline_aligned_in_smp;
+ struct idr aidr; /* ATID id space */
+ struct idr hwtid_idr;
+ struct idr stid_idr;
+
+ spinlock_t idr_lock cacheline_aligned_in_smp;
+
+ struct net_device *egr_dev[NCHAN * 2];
+ struct sk_buff *rspq_skb_cache[1 << RSPQ_HASH_BITS];
+ struct sk_buff *askb;
+
+ struct sk_buff_head deferq;
+ struct work_struct deferq_task;
+
+ struct list_head list_node;
+ struct list_head rcu_node;
+ struct list_head na_node;
+
[Crypto v6 07/12] chcr: Key Macro
Define macro for TLS Key context
Signed-off-by: Atul Gupta
---
drivers/crypto/chelsio/chcr_algo.h | 42 +
drivers/crypto/chelsio/chcr_core.h | 55 +-
2 files changed, 96 insertions(+), 1 deletion(-)
diff --git a/drivers/crypto/chelsio/chcr_algo.h
b/drivers/crypto/chelsio/chcr_algo.h
index d1673a5..f263cd4 100644
--- a/drivers/crypto/chelsio/chcr_algo.h
+++ b/drivers/crypto/chelsio/chcr_algo.h
@@ -86,6 +86,39 @@
KEY_CONTEXT_OPAD_PRESENT_M)
#define KEY_CONTEXT_OPAD_PRESENT_F KEY_CONTEXT_OPAD_PRESENT_V(1U)
+#define TLS_KEYCTX_RXFLIT_CNT_S 24
+#define TLS_KEYCTX_RXFLIT_CNT_V(x) ((x) << TLS_KEYCTX_RXFLIT_CNT_S)
+
+#define TLS_KEYCTX_RXPROT_VER_S 20
+#define TLS_KEYCTX_RXPROT_VER_M 0xf
+#define TLS_KEYCTX_RXPROT_VER_V(x) ((x) << TLS_KEYCTX_RXPROT_VER_S)
+
+#define TLS_KEYCTX_RXCIPH_MODE_S 16
+#define TLS_KEYCTX_RXCIPH_MODE_M 0xf
+#define TLS_KEYCTX_RXCIPH_MODE_V(x) ((x) << TLS_KEYCTX_RXCIPH_MODE_S)
+
+#define TLS_KEYCTX_RXAUTH_MODE_S 12
+#define TLS_KEYCTX_RXAUTH_MODE_M 0xf
+#define TLS_KEYCTX_RXAUTH_MODE_V(x) ((x) << TLS_KEYCTX_RXAUTH_MODE_S)
+
+#define TLS_KEYCTX_RXCIAU_CTRL_S 11
+#define TLS_KEYCTX_RXCIAU_CTRL_V(x) ((x) << TLS_KEYCTX_RXCIAU_CTRL_S)
+
+#define TLS_KEYCTX_RX_SEQCTR_S 9
+#define TLS_KEYCTX_RX_SEQCTR_M 0x3
+#define TLS_KEYCTX_RX_SEQCTR_V(x) ((x) << TLS_KEYCTX_RX_SEQCTR_S)
+
+#define TLS_KEYCTX_RX_VALID_S 8
+#define TLS_KEYCTX_RX_VALID_V(x) ((x) << TLS_KEYCTX_RX_VALID_S)
+
+#define TLS_KEYCTX_RXCK_SIZE_S 3
+#define TLS_KEYCTX_RXCK_SIZE_M 0x7
+#define TLS_KEYCTX_RXCK_SIZE_V(x) ((x) << TLS_KEYCTX_RXCK_SIZE_S)
+
+#define TLS_KEYCTX_RXMK_SIZE_S 0
+#define TLS_KEYCTX_RXMK_SIZE_M 0x7
+#define TLS_KEYCTX_RXMK_SIZE_V(x) ((x) << TLS_KEYCTX_RXMK_SIZE_S)
+
#define CHCR_HASH_MAX_DIGEST_SIZE 64
#define CHCR_MAX_SHA_DIGEST_SIZE 64
@@ -176,6 +209,15 @@
KEY_CONTEXT_SALT_PRESENT_V(1) | \
KEY_CONTEXT_CTX_LEN_V((ctx_len)))
+#define FILL_KEY_CRX_HDR(ck_size, mk_size, d_ck, opad, ctx_len) \
+ htonl(TLS_KEYCTX_RXMK_SIZE_V(mk_size) | \
+ TLS_KEYCTX_RXCK_SIZE_V(ck_size) | \
+ TLS_KEYCTX_RX_VALID_V(1) | \
+ TLS_KEYCTX_RX_SEQCTR_V(3) | \
+ TLS_KEYCTX_RXAUTH_MODE_V(4) | \
+ TLS_KEYCTX_RXCIPH_MODE_V(2) | \
+ TLS_KEYCTX_RXFLIT_CNT_V((ctx_len)))
+
#define FILL_WR_OP_CCTX_SIZE \
htonl( \
FW_CRYPTO_LOOKASIDE_WR_OPCODE_V( \
diff --git a/drivers/crypto/chelsio/chcr_core.h
b/drivers/crypto/chelsio/chcr_core.h
index 3c29ee0..77056a9 100644
--- a/drivers/crypto/chelsio/chcr_core.h
+++ b/drivers/crypto/chelsio/chcr_core.h
@@ -65,10 +65,58 @@
struct _key_ctx {
__be32 ctx_hdr;
u8 salt[MAX_SALT];
- __be64 reserverd;
+ __be64 iv_to_auth;
unsigned char key[0];
};
+#define KEYCTX_TX_WR_IV_S 55
+#define KEYCTX_TX_WR_IV_M 0x1ffULL
+#define KEYCTX_TX_WR_IV_V(x) ((x) << KEYCTX_TX_WR_IV_S)
+#define KEYCTX_TX_WR_IV_G(x) \
+ (((x) >> KEYCTX_TX_WR_IV_S) & KEYCTX_TX_WR_IV_M)
+
+#define KEYCTX_TX_WR_AAD_S 47
+#define KEYCTX_TX_WR_AAD_M 0xffULL
+#define KEYCTX_TX_WR_AAD_V(x) ((x) << KEYCTX_TX_WR_AAD_S)
+#define KEYCTX_TX_WR_AAD_G(x) (((x) >> KEYCTX_TX_WR_AAD_S) & \
+ KEYCTX_TX_WR_AAD_M)
+
+#define KEYCTX_TX_WR_AADST_S 39
+#define KEYCTX_TX_WR_AADST_M 0xffULL
+#define KEYCTX_TX_WR_AADST_V(x) ((x) << KEYCTX_TX_WR_AADST_S)
+#define KEYCTX_TX_WR_AADST_G(x) \
+ (((x) >> KEYCTX_TX_WR_AADST_S) & KEYCTX_TX_WR_AADST_M)
+
+#define KEYCTX_TX_WR_CIPHER_S 30
+#define KEYCTX_TX_WR_CIPHER_M 0x1ffULL
+#define KEYCTX_TX_WR_CIPHER_V(x) ((x) << KEYCTX_TX_WR_CIPHER_S)
+#define KEYCTX_TX_WR_CIPHER_G(x) \
+ (((x) >> KEYCTX_TX_WR_CIPHER_S) & KEYCTX_TX_WR_CIPHER_M)
+
+#define KEYCTX_TX_WR_CIPHERST_S 23
+#define KEYCTX_TX_WR_CIPHERST_M 0x7f
+#define KEYCTX_TX_WR_CIPHERST_V(x) ((x) << KEYCTX_TX_WR_CIPHERST_S)
+#define KEYCTX_TX_WR_CIPHERST_G(x) \
+ (((x) >> KEYCTX_TX_WR_CIPHERST_S) & KEYCTX_TX_WR_CIPHERST_M)
+
+#define KEYCTX_TX_WR_AUTH_S 14
+#define KEYCTX_TX_WR_AUTH_M 0x1ff
+#define KEYCTX_TX_WR_AUTH_V(x) ((x) << KEYCTX_TX_WR_AUTH_S)
+#define KEYCTX_TX_WR_AUTH_G(x) \
+ (((x) >> KEYCTX_TX_WR_AUTH_S) & KEYCTX_TX_WR_AUTH_M)
+
+#define KEYCTX_TX_WR_AUTHST_S 7
+#define KEYCTX_TX_WR_AUTHST_M 0x7f
+#define KEYCTX_TX_WR_AUTHST_V(x) ((x) << KEYCTX_TX_WR_AUTHST_S)
+#define KEYCTX_TX_WR_AUTHST_G(x) \
+ (((x) >> KEYCTX_TX_WR_AUTHST_S) & KEYCTX_TX_WR_AUTHST_M)
+
+#define KEYCTX_TX_WR_AUTHIN_S 0
+#define KEYCTX_TX_WR_AUTHIN_M 0x7f
+#define KEYCTX_TX_WR_AUTHIN_V(x) ((x) << KEYCTX_TX_WR_AUTHIN_S)
+#define KEYCTX_TX_WR_AUTHIN_G(x) \
+ (((x) >> KEYCTX_TX_WR_AUTHIN_S) & KEYCTX_TX_WR_AUTH
[Crypto v6 11/12] chtls: Register chtls Inline TLS with net tls
Register chtls as Inline TLS driver, chtls is ULD to cxgb4.
Setsockopt to program (tx/rx) keys on chip. Support AES GCM
of key size 128. Support both Inline Rx and Tx.
Signed-off-by: Atul Gupta
---
drivers/crypto/chelsio/chtls/chtls_main.c | 600 ++
include/uapi/linux/tls.h | 1 +
2 files changed, 601 insertions(+)
create mode 100644 drivers/crypto/chelsio/chtls/chtls_main.c
diff --git a/drivers/crypto/chelsio/chtls/chtls_main.c
b/drivers/crypto/chelsio/chtls/chtls_main.c
new file mode 100644
index 000..657c515
--- /dev/null
+++ b/drivers/crypto/chelsio/chtls/chtls_main.c
@@ -0,0 +1,600 @@
+/*
+ * Copyright (c) 2017 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Written by: Atul Gupta (atul.gu...@chelsio.com)
+ */
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "chtls.h"
+#include "chtls_cm.h"
+
+#define DRV_NAME "chtls"
+
+/*
+ * chtls device management
+ * maintains a list of the chtls devices
+ */
+static LIST_HEAD(cdev_list);
+static DEFINE_MUTEX(cdev_mutex);
+static DEFINE_MUTEX(cdev_list_lock);
+
+static struct proto chtls_cpl_prot;
+static struct proto chtls_base_prot;
+static DEFINE_MUTEX(notify_mutex);
+static RAW_NOTIFIER_HEAD(listen_notify_list);
+struct request_sock_ops chtls_rsk_ops;
+static uint send_page_order = (14 - PAGE_SHIFT < 0) ? 0 : 14 - PAGE_SHIFT;
+
+static int register_listen_notifier(struct notifier_block *nb)
+{
+ int err;
+
+ mutex_lock(¬ify_mutex);
+ err = raw_notifier_chain_register(&listen_notify_list, nb);
+ mutex_unlock(¬ify_mutex);
+ return err;
+}
+
+static int unregister_listen_notifier(struct notifier_block *nb)
+{
+ int err;
+
+ mutex_lock(¬ify_mutex);
+ err = raw_notifier_chain_unregister(&listen_notify_list, nb);
+ mutex_unlock(¬ify_mutex);
+ return err;
+}
+
+static int listen_notify_handler(struct notifier_block *this,
+unsigned long event, void *data)
+{
+ struct sock *sk = data;
+ struct chtls_dev *cdev;
+ int ret = NOTIFY_DONE;
+
+ switch (event) {
+ case CHTLS_LISTEN_START:
+ case CHTLS_LISTEN_STOP:
+ mutex_lock(&cdev_list_lock);
+ list_for_each_entry(cdev, &cdev_list, list) {
+ if (event == CHTLS_LISTEN_START)
+ ret = chtls_listen_start(cdev, sk);
+ else
+ chtls_listen_stop(cdev, sk);
+ }
+ mutex_unlock(&cdev_list_lock);
+ break;
+ }
+ return ret;
+}
+
+static struct notifier_block listen_notifier = {
+ .notifier_call = listen_notify_handler
+};
+
+static int listen_backlog_rcv(struct sock *sk, struct sk_buff *skb)
+{
+ if (likely(skb_transport_header(skb) != skb_network_header(skb)))
+ return tcp_v4_do_rcv(sk, skb);
+ BLOG_SKB_CB(skb)->backlog_rcv(sk, skb);
+ return 0;
+}
+
+static int chtls_start_listen(struct sock *sk)
+{
+ int err;
+
+ if (sk->sk_protocol != IPPROTO_TCP)
+ return -EPROTONOSUPPORT;
+
+ if (sk->sk_family == PF_INET &&
+ LOOPBACK(inet_sk(sk)->inet_rcv_saddr))
+ return -EADDRNOTAVAIL;
+
+ sk->sk_backlog_rcv = listen_backlog_rcv;
+ mutex_lock(¬ify_mutex);
+ err = raw_notifier_call_chain(&listen_notify_list,
+ CHTLS_LISTEN_START, sk);
+ mutex_unlock(¬ify_mutex);
+ return err;
+}
+
+static int chtls_hash(struct sock *sk)
+{
+ int err;
+
+ err = tcp_prot.hash(sk);
+ if (sk->sk_state == TCP_LISTEN)
+ err |= chtls_start_listen(sk);
+
+ if (err)
+ tcp_prot.unhash(sk);
+ return err;
+}
+
+static int chtls_stop_listen(struct sock *sk)
+{
+ if (sk->sk_protocol != IPPROTO_TCP)
+ return -EPROTONOSUPPORT;
+
+ mutex_lock(¬ify_mutex);
+ raw_notifier_call_chain(&listen_notify_list,
+ CHTLS_LISTEN_STOP, sk);
+ mutex_unlock(¬ify_mutex);
+ return 0;
+}
+
+static void chtls_unhash(struct sock *sk)
+{
+ if (sk->sk_state == TCP_LISTEN)
+ chtls_stop_listen(sk);
+ tcp_prot.unhash(sk);
+}
+
+static int chtls_netdev(struct tls_device *dev,
+ struct net_device *netdev)
+{
+ struct chtls_dev *cdev = to_chtls_dev(dev);
+ int i;
+
+ for (i = 0; i < cdev->lldi->nports; i++)
+ if (cdev->ports[i] == netdev)
+ return 1;
+
+ return 0;
+}
+
+sta
[Crypto v6 08/12] chtls: Key program
Program the tx and rx key on chip.
Signed-off-by: Atul Gupta
---
drivers/crypto/chelsio/chtls/chtls_hw.c | 394
1 file changed, 394 insertions(+)
create mode 100644 drivers/crypto/chelsio/chtls/chtls_hw.c
diff --git a/drivers/crypto/chelsio/chtls/chtls_hw.c
b/drivers/crypto/chelsio/chtls/chtls_hw.c
new file mode 100644
index 000..c3e17159
--- /dev/null
+++ b/drivers/crypto/chelsio/chtls/chtls_hw.c
@@ -0,0 +1,394 @@
+/*
+ * Copyright (c) 2017 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Written by: Atul Gupta (atul.gu...@chelsio.com)
+ */
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "chtls.h"
+#include "chtls_cm.h"
+
+static void __set_tcb_field_direct(struct chtls_sock *csk,
+ struct cpl_set_tcb_field *req, u16 word,
+ u64 mask, u64 val, u8 cookie, int no_reply)
+{
+ struct ulptx_idata *sc;
+
+ INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, csk->tid);
+ req->wr.wr_mid |= htonl(FW_WR_FLOWID_V(csk->tid));
+ req->reply_ctrl = htons(NO_REPLY_V(no_reply) |
+ QUEUENO_V(csk->rss_qid));
+ req->word_cookie = htons(TCB_WORD_V(word) | TCB_COOKIE_V(cookie));
+ req->mask = cpu_to_be64(mask);
+ req->val = cpu_to_be64(val);
+ sc = (struct ulptx_idata *)(req + 1);
+ sc->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP));
+ sc->len = htonl(0);
+}
+
+static void __set_tcb_field(struct sock *sk, struct sk_buff *skb, u16 word,
+ u64 mask, u64 val, u8 cookie, int no_reply)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+ struct cpl_set_tcb_field *req;
+ struct ulptx_idata *sc;
+ unsigned int wrlen = roundup(sizeof(*req) + sizeof(*sc), 16);
+
+ req = (struct cpl_set_tcb_field *)__skb_put(skb, wrlen);
+ __set_tcb_field_direct(csk, req, word, mask, val, cookie, no_reply);
+ set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id);
+}
+
+static int chtls_set_tcb_field(struct sock *sk, u16 word, u64 mask, u64 val)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+ struct sk_buff *skb;
+ struct cpl_set_tcb_field *req;
+ struct ulptx_idata *sc;
+ unsigned int wrlen = roundup(sizeof(*req) + sizeof(*sc), 16);
+ unsigned int credits_needed = DIV_ROUND_UP(wrlen, 16);
+
+ skb = alloc_skb(wrlen, GFP_ATOMIC);
+ if (!skb)
+ return -ENOMEM;
+
+ __set_tcb_field(sk, skb, word, mask, val, 0, 1);
+ set_queue(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA, sk);
+ csk->wr_credits -= credits_needed;
+ csk->wr_unacked += credits_needed;
+ enqueue_wr(csk, skb);
+ cxgb4_ofld_send(csk->egress_dev, skb);
+ return 0;
+}
+
+/*
+ * Set one of the t_flags bits in the TCB.
+ */
+int chtls_set_tcb_tflag(struct sock *sk, unsigned int bit_pos, int val)
+{
+ return chtls_set_tcb_field(sk, 1, 1ULL << bit_pos,
+ val << bit_pos);
+}
+
+static int chtls_set_tcb_keyid(struct sock *sk, int keyid)
+{
+ return chtls_set_tcb_field(sk, 31, 0xULL, keyid);
+}
+
+static int chtls_set_tcb_seqno(struct sock *sk)
+{
+ return chtls_set_tcb_field(sk, 28, ~0ULL, 0);
+}
+
+static int chtls_set_tcb_quiesce(struct sock *sk, int val)
+{
+ return chtls_set_tcb_field(sk, 1, (1ULL << TF_RX_QUIESCE_S),
+ TF_RX_QUIESCE_V(val));
+}
+
+static void *chtls_alloc_mem(unsigned long size)
+{
+ void *p = kmalloc(size, GFP_KERNEL);
+
+ if (!p)
+ p = vmalloc(size);
+ if (p)
+ memset(p, 0, size);
+ return p;
+}
+
+static void chtls_free_mem(void *addr)
+{
+ unsigned long p = (unsigned long)addr;
+
+ if (p >= VMALLOC_START && p < VMALLOC_END)
+ vfree(addr);
+ else
+ kfree(addr);
+}
+
+/* TLS Key bitmap processing */
+int chtls_init_kmap(struct chtls_dev *cdev, struct cxgb4_lld_info *lldi)
+{
+ unsigned int num_key_ctx, bsize;
+
+ num_key_ctx = (lldi->vr->key.size / TLS_KEY_CONTEXT_SZ);
+ bsize = BITS_TO_LONGS(num_key_ctx);
+
+ cdev->kmap.size = num_key_ctx;
+ cdev->kmap.available = bsize;
+ cdev->kmap.addr = chtls_alloc_mem(sizeof(*cdev->kmap.addr) *
+ bsize);
+ if (!cdev->kmap.addr)
+ return -1;
+
+ cdev->kmap.start = lldi->vr->key.start;
+ spin_lock_init(&cdev->kmap.lock);
+ return 0;
+}
+
+void chtls_free_kmap(st
[Crypto v6 09/12] chtls: CPL handler definition
CPL handlers for TLS session, record transmit and receive.
Signed-off-by: Atul Gupta
---
drivers/crypto/chelsio/chtls/chtls_cm.c | 2041 +++
net/ipv4/tcp_minisocks.c|1 +
2 files changed, 2042 insertions(+)
create mode 100644 drivers/crypto/chelsio/chtls/chtls_cm.c
diff --git a/drivers/crypto/chelsio/chtls/chtls_cm.c
b/drivers/crypto/chelsio/chtls/chtls_cm.c
new file mode 100644
index 000..1c95e87
--- /dev/null
+++ b/drivers/crypto/chelsio/chtls/chtls_cm.c
@@ -0,0 +1,2041 @@
+/*
+ * Copyright (c) 2017 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Written by: Atul Gupta (atul.gu...@chelsio.com)
+ */
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "chtls.h"
+#include "chtls_cm.h"
+
+extern struct request_sock_ops chtls_rsk_ops;
+
+/*
+ * State transitions and actions for close. Note that if we are in SYN_SENT
+ * we remain in that state as we cannot control a connection while it's in
+ * SYN_SENT; such connections are allowed to establish and are then aborted.
+ */
+static unsigned char new_state[16] = {
+ /* current state: new state: action: */
+ /* (Invalid) */ TCP_CLOSE,
+ /* TCP_ESTABLISHED */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
+ /* TCP_SYN_SENT*/ TCP_SYN_SENT,
+ /* TCP_SYN_RECV*/ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
+ /* TCP_FIN_WAIT1 */ TCP_FIN_WAIT1,
+ /* TCP_FIN_WAIT2 */ TCP_FIN_WAIT2,
+ /* TCP_TIME_WAIT */ TCP_CLOSE,
+ /* TCP_CLOSE */ TCP_CLOSE,
+ /* TCP_CLOSE_WAIT */ TCP_LAST_ACK | TCP_ACTION_FIN,
+ /* TCP_LAST_ACK*/ TCP_LAST_ACK,
+ /* TCP_LISTEN */ TCP_CLOSE,
+ /* TCP_CLOSING */ TCP_CLOSING,
+};
+
+static struct chtls_sock *chtls_sock_create(struct chtls_dev *cdev)
+{
+ struct chtls_sock *csk = kzalloc(sizeof(*csk), GFP_ATOMIC);
+
+ if (!csk)
+ return NULL;
+
+ csk->txdata_skb_cache = alloc_skb(TXDATA_SKB_LEN, GFP_ATOMIC);
+ if (!csk->txdata_skb_cache) {
+ kfree(csk);
+ return NULL;
+ }
+
+ kref_init(&csk->kref);
+ csk->cdev = cdev;
+ skb_queue_head_init(&csk->txq);
+ csk->wr_skb_head = NULL;
+ csk->wr_skb_tail = NULL;
+ csk->mss = MAX_MSS;
+ csk->tlshws.ofld = 1;
+ csk->tlshws.txkey = -1;
+ csk->tlshws.rxkey = -1;
+ csk->tlshws.mfs = TLS_MFS;
+ skb_queue_head_init(&csk->tlshws.sk_recv_queue);
+ return csk;
+}
+
+static void chtls_sock_release(struct kref *ref)
+{
+ struct chtls_sock *csk =
+ container_of(ref, struct chtls_sock, kref);
+
+ kfree(csk);
+}
+
+static struct net_device *chtls_ipv4_netdev(struct chtls_dev *cdev,
+ struct sock *sk)
+{
+ struct net_device *ndev = cdev->ports[0];
+
+ if (likely(!inet_sk(sk)->inet_rcv_saddr))
+ return ndev;
+
+ ndev = ip_dev_find(&init_net, inet_sk(sk)->inet_rcv_saddr);
+ if (!ndev)
+ return NULL;
+
+ if (is_vlan_dev(ndev))
+ return vlan_dev_real_dev(ndev);
+ return ndev;
+}
+
+static void assign_rxopt(struct sock *sk, unsigned int opt)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+ struct tcp_sock *tp = tcp_sk(sk);
+ const struct chtls_dev *cdev;
+
+ cdev = csk->cdev;
+ tp->tcp_header_len = sizeof(struct tcphdr);
+ tp->rx_opt.mss_clamp = cdev->mtus[TCPOPT_MSS_G(opt)] - 40;
+ tp->mss_cache= tp->rx_opt.mss_clamp;
+ tp->rx_opt.tstamp_ok = TCPOPT_TSTAMP_G(opt);
+ tp->rx_opt.snd_wscale= TCPOPT_SACK_G(opt);
+ tp->rx_opt.wscale_ok = TCPOPT_WSCALE_OK_G(opt);
+ SND_WSCALE(tp) = TCPOPT_SND_WSCALE_G(opt);
+ if (!tp->rx_opt.wscale_ok)
+ tp->rx_opt.rcv_wscale = 0;
+ if (tp->rx_opt.tstamp_ok) {
+ tp->tcp_header_len += TCPOLEN_TSTAMP_ALIGNED;
+ tp->rx_opt.mss_clamp -= TCPOLEN_TSTAMP_ALIGNED;
+ } else if (csk->opt2 & TSTAMPS_EN_F) {
+ csk->opt2 &= ~TSTAMPS_EN_F;
+ csk->mtu_idx = TCPOPT_MSS_G(opt);
+ }
+}
+
+static void chtls_purge_rcv_queue(struct sock *sk)
+{
+ struct sk_buff *skb;
+
+ while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) {
+ skb_dst_set(skb, (void *)NULL);
+ kfree_skb(skb);
+ }
+}
+
+static void chtls_purge_write_queue(struct s
[Crypto v6 03/12] tls: support for inline tls
Facility to register Inline TLS drivers to net/tls. Setup
TLS_FULL_HW prot to listen on offload device.
Cases handled
1. Inline TLS device exists, setup prot for TLS_FULL_HW
2. Atleast one Inline TLS exists, sets TLS_FULL_HW. If
non-inline capable device establish connection, move to TLS_SW_TX
3. default mode TLS_SW_TX continues
Signed-off-by: Atul Gupta
---
net/tls/tls_main.c | 123 ++---
1 file changed, 116 insertions(+), 7 deletions(-)
diff --git a/net/tls/tls_main.c b/net/tls/tls_main.c
index b0d5fce..88eafec 100644
--- a/net/tls/tls_main.c
+++ b/net/tls/tls_main.c
@@ -38,6 +38,7 @@
#include
#include
#include
+#include
#include
@@ -45,13 +46,9 @@
MODULE_DESCRIPTION("Transport Layer Security Support");
MODULE_LICENSE("Dual BSD/GPL");
-enum {
- TLS_BASE_TX,
- TLS_SW_TX,
- TLS_NUM_CONFIG,
-};
-
-static struct proto tls_prots[TLS_NUM_CONFIG];
+static LIST_HEAD(device_list);
+static DEFINE_MUTEX(device_mutex);
+struct proto tls_prots[TLS_NUM_CONFIG];
static inline void update_sk_prot(struct sock *sk, struct tls_context *ctx)
{
@@ -260,6 +257,37 @@ static void tls_sk_proto_close(struct sock *sk, long
timeout)
sk_proto_close(sk, timeout);
}
+static struct net_device *get_netdev(struct sock *sk)
+{
+ struct inet_sock *inet = inet_sk(sk);
+ struct net_device *netdev = NULL;
+
+ netdev = dev_get_by_index(sock_net(sk), inet->cork.fl.flowi_oif);
+ return netdev;
+}
+
+static int get_tls_offload_dev(struct sock *sk)
+{
+ struct net_device *netdev;
+ struct tls_device *dev;
+ int rc = 0;
+
+ netdev = get_netdev(sk);
+ if (!netdev)
+ return -EINVAL;
+
+ mutex_lock(&device_mutex);
+ list_for_each_entry(dev, &device_list, dev_list) {
+ if (dev->netdev && dev->netdev(dev, netdev)) {
+ rc = -EEXIST;
+ break;
+ }
+ }
+ mutex_unlock(&device_mutex);
+ dev_put(netdev);
+ return rc;
+}
+
static int do_tls_getsockopt_tx(struct sock *sk, char __user *optval,
int __user *optlen)
{
@@ -403,6 +431,15 @@ static int do_tls_setsockopt_tx(struct sock *sk, char
__user *optval,
goto err_crypto_info;
}
+ rc = get_tls_offload_dev(sk);
+ if (rc == -EINVAL) {
+ goto out;
+ } else if (rc == -EEXIST) {
+ /* Retain HW unhash for cleanup and move to SW Tx */
+ sk->sk_prot[TLS_BASE_TX].unhash =
+ sk->sk_prot[TLS_FULL_HW].unhash;
+ }
+
/* currently SW is default, we will have ethtool in future */
rc = tls_set_sw_offload(sk, ctx);
tx_conf = TLS_SW_TX;
@@ -450,6 +487,54 @@ static int tls_setsockopt(struct sock *sk, int level, int
optname,
return do_tls_setsockopt(sk, optname, optval, optlen);
}
+static int tls_hw_prot(struct sock *sk)
+{
+ struct tls_context *ctx = tls_get_ctx(sk);
+ struct tls_device *dev;
+
+ mutex_lock(&device_mutex);
+ list_for_each_entry(dev, &device_list, dev_list) {
+ if (dev->feature && dev->feature(dev)) {
+ ctx->tx_conf = TLS_FULL_HW;
+ update_sk_prot(sk, ctx);
+ break;
+ }
+ }
+ mutex_unlock(&device_mutex);
+ return ctx->tx_conf;
+}
+
+static void tls_hw_unhash(struct sock *sk)
+{
+ struct tls_device *dev;
+
+ mutex_lock(&device_mutex);
+ list_for_each_entry(dev, &device_list, dev_list) {
+ if (dev->unhash)
+ dev->unhash(dev, sk);
+ }
+ mutex_unlock(&device_mutex);
+ tcp_prot.unhash(sk);
+}
+
+static int tls_hw_hash(struct sock *sk)
+{
+ struct tls_device *dev;
+ int err;
+
+ err = tcp_prot.hash(sk);
+ mutex_lock(&device_mutex);
+ list_for_each_entry(dev, &device_list, dev_list) {
+ if (dev->hash)
+ err |= dev->hash(dev, sk);
+ }
+ mutex_unlock(&device_mutex);
+
+ if (err)
+ tls_hw_unhash(sk);
+ return err;
+}
+
static int tls_init(struct sock *sk)
{
struct inet_connection_sock *icsk = inet_csk(sk);
@@ -477,6 +562,9 @@ static int tls_init(struct sock *sk)
ctx->sk_proto_close = sk->sk_prot->close;
ctx->tx_conf = TLS_BASE_TX;
+ if (tls_hw_prot(sk) == TLS_FULL_HW)
+ goto out;
+
update_sk_prot(sk, ctx);
out:
return rc;
@@ -500,7 +588,27 @@ static void build_protos(struct proto *prot, struct proto
*base)
prot[TLS_SW_TX] = prot[TLS_BASE_TX];
prot[TLS_SW_TX].sendmsg = tls_sw_sendmsg;
prot[TLS_SW_TX].sendpage= tls_sw_sendpage;
+
[Crypto v6 12/12] Makefile Kconfig
Entry for Inline TLS as another driver dependent on cxgb4 and chcr Signed-off-by: Atul Gupta --- drivers/crypto/chelsio/Kconfig| 11 +++ drivers/crypto/chelsio/Makefile | 1 + drivers/crypto/chelsio/chtls/Makefile | 4 3 files changed, 16 insertions(+) create mode 100644 drivers/crypto/chelsio/chtls/Makefile diff --git a/drivers/crypto/chelsio/Kconfig b/drivers/crypto/chelsio/Kconfig index 5ae9f87..930d82d 100644 --- a/drivers/crypto/chelsio/Kconfig +++ b/drivers/crypto/chelsio/Kconfig @@ -29,3 +29,14 @@ config CHELSIO_IPSEC_INLINE default n ---help--- Enable support for IPSec Tx Inline. + +config CRYPTO_DEV_CHELSIO_TLS +tristate "Chelsio Crypto Inline TLS Driver" +depends on CHELSIO_T4 +depends on TLS +select CRYPTO_DEV_CHELSIO +---help--- + Support Chelsio Inline TLS with Chelsio crypto accelerator. + + To compile this driver as a module, choose M here: the module + will be called chtls. diff --git a/drivers/crypto/chelsio/Makefile b/drivers/crypto/chelsio/Makefile index eaecaf1..639e571 100644 --- a/drivers/crypto/chelsio/Makefile +++ b/drivers/crypto/chelsio/Makefile @@ -3,3 +3,4 @@ ccflags-y := -Idrivers/net/ethernet/chelsio/cxgb4 obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chcr.o chcr-objs := chcr_core.o chcr_algo.o chcr-$(CONFIG_CHELSIO_IPSEC_INLINE) += chcr_ipsec.o +obj-$(CONFIG_CRYPTO_DEV_CHELSIO_TLS) += chtls/ diff --git a/drivers/crypto/chelsio/chtls/Makefile b/drivers/crypto/chelsio/chtls/Makefile new file mode 100644 index 000..df13795 --- /dev/null +++ b/drivers/crypto/chelsio/chtls/Makefile @@ -0,0 +1,4 @@ +ccflags-y := -Idrivers/net/ethernet/chelsio/cxgb4 -Idrivers/crypto/chelsio/ + +obj-$(CONFIG_CRYPTO_DEV_CHELSIO_TLS) += chtls.o +chtls-objs := chtls_main.o chtls_cm.o chtls_io.o chtls_hw.o -- 1.8.3.1
[Crypto v6 10/12] chtls: Inline crypto request Tx/Rx
TLS handler for record transmit and receive.
Create Inline TLS work request and post to FW.
Signed-off-by: Atul Gupta
---
drivers/crypto/chelsio/chtls/chtls_io.c | 1867 +++
1 file changed, 1867 insertions(+)
create mode 100644 drivers/crypto/chelsio/chtls/chtls_io.c
diff --git a/drivers/crypto/chelsio/chtls/chtls_io.c
b/drivers/crypto/chelsio/chtls/chtls_io.c
new file mode 100644
index 000..0c5d6c1
--- /dev/null
+++ b/drivers/crypto/chelsio/chtls/chtls_io.c
@@ -0,0 +1,1867 @@
+/*
+ * Copyright (c) 2017 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Written by: Atul Gupta (atul.gu...@chelsio.com)
+ */
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "chtls.h"
+#include "chtls_cm.h"
+
+static bool is_tls_hw(struct chtls_sock *csk)
+{
+ return csk->tlshws.ofld;
+}
+
+static bool is_tls_rx(struct chtls_sock *csk)
+{
+ return (csk->tlshws.rxkey >= 0);
+}
+
+static bool is_tls_tx(struct chtls_sock *csk)
+{
+ return (csk->tlshws.txkey >= 0);
+}
+
+static bool is_tls_skb(struct chtls_sock *csk, const struct sk_buff *skb)
+{
+ return (is_tls_hw(csk) && skb_ulp_tls_skb_flags(skb));
+}
+
+static int key_size(void *sk)
+{
+ return 16; /* Key on DDR */
+}
+
+#define ceil(x, y) \
+ ({ unsigned long __x = (x), __y = (y); (__x + __y - 1) / __y; })
+
+static int data_sgl_len(const struct sk_buff *skb)
+{
+ unsigned int cnt;
+
+ cnt = skb_shinfo(skb)->nr_frags;
+ return (sgl_len(cnt) * 8);
+}
+
+static int nos_ivs(struct sock *sk, unsigned int size)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+
+ return ceil(size, csk->tlshws.mfs);
+}
+
+#define TLS_WR_CPL_LEN \
+ (sizeof(struct fw_tlstx_data_wr) + \
+ sizeof(struct cpl_tx_tls_sfo))
+
+static int is_ivs_imm(struct sock *sk, const struct sk_buff *skb)
+{
+ int ivs_size = nos_ivs(sk, skb->len) * CIPHER_BLOCK_SIZE;
+ int hlen = TLS_WR_CPL_LEN + data_sgl_len(skb);
+
+ if ((hlen + key_size(sk) + ivs_size) <
+ MAX_IMM_OFLD_TX_DATA_WR_LEN) {
+ ULP_SKB_CB(skb)->ulp.tls.iv = 1;
+ return 1;
+ }
+ ULP_SKB_CB(skb)->ulp.tls.iv = 0;
+ return 0;
+}
+
+static int max_ivs_size(struct sock *sk, int size)
+{
+ return (nos_ivs(sk, size) * CIPHER_BLOCK_SIZE);
+}
+
+static int ivs_size(struct sock *sk, const struct sk_buff *skb)
+{
+ return (is_ivs_imm(sk, skb) ? (nos_ivs(sk, skb->len) *
+CIPHER_BLOCK_SIZE) : 0);
+}
+
+static int flowc_wr_credits(int nparams, int *flowclenp)
+{
+ int flowclen16, flowclen;
+
+ flowclen = offsetof(struct fw_flowc_wr, mnemval[nparams]);
+ flowclen16 = DIV_ROUND_UP(flowclen, 16);
+ flowclen = flowclen16 * 16;
+
+ if (flowclenp)
+ *flowclenp = flowclen;
+
+ return flowclen16;
+}
+
+static struct sk_buff *create_flowc_wr_skb(struct sock *sk,
+ struct fw_flowc_wr *flowc,
+ int flowclen)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+ struct sk_buff *skb;
+
+ skb = alloc_skb(flowclen, GFP_ATOMIC);
+ if (!skb)
+ return NULL;
+
+ memcpy(__skb_put(skb, flowclen), flowc, flowclen);
+ set_queue(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA, sk);
+
+ return skb;
+}
+
+static int send_flowc_wr(struct sock *sk, struct fw_flowc_wr *flowc,
+int flowclen)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+ bool syn_sent = (sk->sk_state == TCP_SYN_SENT);
+ struct tcp_sock *tp = tcp_sk(sk);
+ int flowclen16 = flowclen / 16;
+ struct sk_buff *skb;
+
+ if (csk_flag(sk, CSK_TX_DATA_SENT)) {
+ skb = create_flowc_wr_skb(sk, flowc, flowclen);
+ if (!skb)
+ return -ENOMEM;
+
+ if (syn_sent)
+ __skb_queue_tail(&csk->ooo_queue, skb);
+ else
+ skb_entail(sk, skb,
+ ULPCB_FLAG_NO_HDR | ULPCB_FLAG_NO_APPEND);
+ return 0;
+ }
+
+ if (!syn_sent) {
+ int ret;
+
+ ret = cxgb4_immdata_send(csk->egress_dev,
+csk->txq_idx,
+flowc, flowclen);
+ if (!ret)
+ return flowclen16;
+ }
+ skb = create_flowc_wr_skb(sk, flowc, flowclen);
+ if (!skb)
+ return -
[Crypto v6 05/12] cxgb4: Inline TLS FW Interface
Key area size in hw-config file. CPL struct for TLS request
and response. Work request for Inline TLS.
Signed-off-by: Atul Gupta
---
drivers/net/ethernet/chelsio/cxgb4/t4_msg.h | 121 ++-
drivers/net/ethernet/chelsio/cxgb4/t4_regs.h | 2 +
drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h | 165 +-
3 files changed, 283 insertions(+), 5 deletions(-)
diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h
b/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h
index d0db442..507cb5a 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h
@@ -81,6 +81,7 @@ enum {
CPL_RX_ISCSI_CMP = 0x45,
CPL_TRACE_PKT_T5 = 0x48,
CPL_RX_ISCSI_DDP = 0x49,
+ CPL_RX_TLS_CMP= 0x4E,
CPL_RDMA_READ_REQ = 0x60,
@@ -88,6 +89,7 @@ enum {
CPL_ACT_OPEN_REQ6 = 0x83,
CPL_TX_TLS_PDU =0x88,
+ CPL_TX_TLS_SFO= 0x89,
CPL_TX_SEC_PDU= 0x8A,
CPL_TX_TLS_ACK= 0x8B,
@@ -97,6 +99,7 @@ enum {
CPL_RX_MPS_PKT= 0xAF,
CPL_TRACE_PKT = 0xB0,
+ CPL_TLS_DATA = 0xB1,
CPL_ISCSI_DATA= 0xB2,
CPL_FW4_MSG = 0xC0,
@@ -151,6 +154,7 @@ enum {
ULP_MODE_RDMA = 4,
ULP_MODE_TCPDDP= 5,
ULP_MODE_FCOE = 6,
+ ULP_MODE_TLS = 8,
};
enum {
@@ -1415,6 +1419,14 @@ struct cpl_tx_data {
#define TX_FORCE_S 13
#define TX_FORCE_V(x) ((x) << TX_FORCE_S)
+#define TX_SHOVE_S14
+#define TX_SHOVE_V(x) ((x) << TX_SHOVE_S)
+
+#define TX_ULP_MODE_S10
+#define TX_ULP_MODE_M0x7
+#define TX_ULP_MODE_V(x) ((x) << TX_ULP_MODE_S)
+#define TX_ULP_MODE_G(x) (((x) >> TX_ULP_MODE_S) & TX_ULP_MODE_M)
+
#define T6_TX_FORCE_S 20
#define T6_TX_FORCE_V(x) ((x) << T6_TX_FORCE_S)
#define T6_TX_FORCE_F T6_TX_FORCE_V(1U)
@@ -1429,12 +1441,21 @@ enum {
ULP_TX_SC_NOOP = 0x80,
ULP_TX_SC_IMM = 0x81,
ULP_TX_SC_DSGL = 0x82,
- ULP_TX_SC_ISGL = 0x83
+ ULP_TX_SC_ISGL = 0x83,
+ ULP_TX_SC_MEMRD = 0x86
};
#define ULPTX_CMD_S24
#define ULPTX_CMD_V(x) ((x) << ULPTX_CMD_S)
+#define ULPTX_LEN16_S0
+#define ULPTX_LEN16_M0xFF
+#define ULPTX_LEN16_V(x) ((x) << ULPTX_LEN16_S)
+
+#define ULP_TX_SC_MORE_S 23
+#define ULP_TX_SC_MORE_V(x) ((x) << ULP_TX_SC_MORE_S)
+#define ULP_TX_SC_MORE_F ULP_TX_SC_MORE_V(1U)
+
struct ulptx_sge_pair {
__be32 len[2];
__be64 addr[2];
@@ -2112,4 +2133,102 @@ enum {
X_CPL_RX_MPS_PKT_TYPE_QFC = 1 << 2,
X_CPL_RX_MPS_PKT_TYPE_PTP = 1 << 3
};
+
+struct cpl_tx_tls_sfo {
+ __be32 op_to_seg_len;
+ __be32 pld_len;
+ __be32 type_protover;
+ __be32 r1_lo;
+ __be32 seqno_numivs;
+ __be32 ivgen_hdrlen;
+ __be64 scmd1;
+};
+
+/* cpl_tx_tls_sfo macros */
+#define CPL_TX_TLS_SFO_OPCODE_S 24
+#define CPL_TX_TLS_SFO_OPCODE_V(x) ((x) << CPL_TX_TLS_SFO_OPCODE_S)
+
+#define CPL_TX_TLS_SFO_DATA_TYPE_S 20
+#define CPL_TX_TLS_SFO_DATA_TYPE_V(x) ((x) << CPL_TX_TLS_SFO_DATA_TYPE_S)
+
+#define CPL_TX_TLS_SFO_CPL_LEN_S16
+#define CPL_TX_TLS_SFO_CPL_LEN_V(x) ((x) << CPL_TX_TLS_SFO_CPL_LEN_S)
+
+#define CPL_TX_TLS_SFO_SEG_LEN_S0
+#define CPL_TX_TLS_SFO_SEG_LEN_M0x
+#define CPL_TX_TLS_SFO_SEG_LEN_V(x) ((x) << CPL_TX_TLS_SFO_SEG_LEN_S)
+#define CPL_TX_TLS_SFO_SEG_LEN_G(x) \
+ (((x) >> CPL_TX_TLS_SFO_SEG_LEN_S) & CPL_TX_TLS_SFO_SEG_LEN_M)
+
+#define CPL_TX_TLS_SFO_TYPE_S 24
+#define CPL_TX_TLS_SFO_TYPE_M 0xff
+#define CPL_TX_TLS_SFO_TYPE_V(x)((x) << CPL_TX_TLS_SFO_TYPE_S)
+#define CPL_TX_TLS_SFO_TYPE_G(x)\
+ (((x) >> CPL_TX_TLS_SFO_TYPE_S) & CPL_TX_TLS_SFO_TYPE_M)
+
+#define CPL_TX_TLS_SFO_PROTOVER_S 8
+#define CPL_TX_TLS_SFO_PROTOVER_M 0x
+#define CPL_TX_TLS_SFO_PROTOVER_V(x)((x) << CPL_TX_TLS_SFO_PROTOVER_S)
+#define CPL_TX_TLS_SFO_PROTOVER_G(x)\
+ (((x) >> CPL_TX_TLS_SFO_PROTOVER_S) & CPL_TX_TLS_SFO_PROTOVER_M)
+
+struct cpl_tls_data {
+ struct rss_header rsshdr;
+ union opcode_tid ot;
+ __be32 length_pkd;
+ __be32 seq;
+ __be32 r1;
+};
+
+#define CPL_TLS_DATA_OPCODE_S 24
+#define CPL_TLS_DATA_OPCODE_M 0xff
+#define CPL_TLS_DATA_OPCODE_V(x)((x) << CPL_TLS_DATA_OPCODE_S)
+#define CPL_TLS_DATA_OPCODE_G(x)\
+ (((x) >> CPL_TLS_DATA_OPCODE_S) & CPL_TLS_DATA_OPCODE_M)
+
+#define CPL_TLS_DATA_TID_S 0
+#define CPL_TLS_DATA_TID_M 0xff
+#define CPL_TLS_DATA_TID_V(x) ((x) << CPL_TLS_DATA_TID_S)
+#define CPL_TLS_DATA_TID_G(x) \
+ (((x) >> CPL_TLS_DATA_TID_S) &
RE: [Crypto v6 03/12] tls: support for inline tls
-Original Message-
From: linux-crypto-ow...@vger.kernel.org
[mailto:linux-crypto-ow...@vger.kernel.org] On Behalf Of David Miller
Sent: Tuesday, February 20, 2018 9:46 PM
To: Atul Gupta
Cc: davejwat...@fb.com; herb...@gondor.apana.org.au; s...@queasysnail.net;
linux-crypto@vger.kernel.org; net...@vger.kernel.org; Ganesh GR
Subject: Re: [Crypto v6 03/12] tls: support for inline tls
From: Atul Gupta
Date: Mon, 19 Feb 2018 12:19:41 +0530
> + struct net_device *netdev = NULL;
> +
> + netdev = dev_get_by_index(sock_net(sk), inet->cork.fl.flowi_oif);
No need for an assignment in the variable declaration here.
You immediately set it to something else unconditionally.
Sure.
> +static int get_tls_offload_dev(struct sock *sk) {
> + struct net_device *netdev;
> + struct tls_device *dev;
> + int rc = 0;
> +
> + netdev = get_netdev(sk);
> + if (!netdev)
> + return -EINVAL;
> +
> + mutex_lock(&device_mutex);
> + list_for_each_entry(dev, &device_list, dev_list) {
> + if (dev->netdev && dev->netdev(dev, netdev)) {
> + rc = -EEXIST;
> + break;
> + }
> + }
> + mutex_unlock(&device_mutex);
> + dev_put(netdev);
> + return rc;
> +}
This is really a confusing function.
It's name suggests that it "gets" the offload device. In that case, if it is
found it should return success. Instead we get an -EEXIST error in that case.
And it returns 0 if not found.
Will rename to no_tls_offload_dev, return 0 is used as default condition in
calling function.
Better to make this do what it says it does, which would be to return '0' when
the device is found and return -ENODEV when it is not found.
> + tcp_prot.unhash(sk);
Done
Do not force this to the ipv4 TCP instance, use the pointer through the socket
to call the proper unhash method.
> + err = tcp_prot.hash(sk);
Done
Likewise.
Thanks
[Crypto v7 00/12] Chelsio Inline TLS
Series for Chelsio Inline TLS driver (chtls.ko)
Use tls ULP infrastructure to register chtls as Inline TLS driver.
Chtls use TCP Sockets to transmit and receive TLS record. TCP proto_ops is
extended to offload TLS record.
T6 adapter provides the following features:
-TLS record offload, TLS header, encrypt, digest and transmit
-TLS record receive and decrypt
-TLS keys store
-TCP/IP engine
-TLS engine
-GCM crypto engine [support CBC also]
TLS provides security at the transport layer. It uses TCP to provide reliable
end-to-end transport of application data. It relies on TCP for any
retransmission. TLS session comprises of three parts:
a. TCP/IP connection
b. TLS handshake
c. Record layer processing
TLS handshake state machine is executed in host (refer standard implementation
eg. OpenSSL). Setsockopt [SOL_TCP, TCP_ULP] initialize TCP proto-ops for
Chelsio inline tls support. setsockopt(sock, SOL_TCP, TCP_ULP, "tls",
sizeof("tls"));
Tx and Rx Keys are decided during handshake and programmed onto the chip after
CCS is exchanged.
struct tls12_crypto_info_aes_gcm_128 crypto_info setsockopt(sock, SOL_TLS,
TLS_TX, &crypto_info, sizeof(crypto_info)) Finish is the first
encrypted/decrypted message tx/rx inline.
On the Tx path TLS engine receive plain text from openssl, insert IV, fetches
the tx key, create cipher text records and generate MAC. TLS header is added to
cipher text and forward to TCP/IP engine for transport layer processing and
transmission on wire.
TX:
Application--openssl--chtls---TLS engine---encrypt/auth---TCP/IP engine---wire.
On the Rx side, data received is PDU aligned at record boundaries. TLS
processes only the complete record. If rx key is programmed on CCS receive,
data is decrypted and plain text is posted to host.
RX:
Wire--cipher-text--TCP/IP engine [PDU align]---TLS engine---
decrypt/auth---plain-text--chtls--openssl--application
v7: func name change, use sk->sk_prot where required
v6: modify prot only for FULL_HW
-corrected commit message for patch 11
v5: set TLS_FULL_HW for registered inline tls drivers
-set TLS_FULL_HW prot for offload connection else move
to TLS_SW_TX
-Case handled for interface with same IP [Dave Miller]
-Removed Specific IP and INADDR_ANY handling [v4]
v4: removed chtls ULP type, retained tls ULP
-registered chtls with net tls
-defined struct tls_device to register the Inline drivers
-ethtool interface tls-inline to enable Inline TLS for interface
-prot update to support inline TLS
v3: fixed the kbuild test issues
-made few funtions static
-initialized few variables
v2: fixed the following based on the review comments of Stephan Mueller,
Stefano Brivio and Hannes Frederic
-Added more details in cover letter
-Fixed indentation and formating issues
-Using aes instead of aes-generic
-memset key info after programing the key on chip
-reordered the patch sequence
Atul Gupta (12):
tls: tls_device struct to register TLS drivers
ethtool: enable Inline TLS in HW
tls: support for inline tls
chtls: structure and macro definiton
cxgb4: Inline TLS FW Interface
cxgb4: LLD driver changes to enable TLS
chcr: Key Macro
chtls: Key program
chtls: CPL handler definition
chtls: Inline crypto request Tx/Rx
chtls: Register chtls Inline TLS with net tls
Makefile Kconfig
drivers/crypto/chelsio/Kconfig | 11 +
drivers/crypto/chelsio/Makefile |1 +
drivers/crypto/chelsio/chcr_algo.h | 42 +
drivers/crypto/chelsio/chcr_core.h | 55 +-
drivers/crypto/chelsio/chtls/Makefile |4 +
drivers/crypto/chelsio/chtls/chtls.h| 487 ++
drivers/crypto/chelsio/chtls/chtls_cm.c | 2041 +++
drivers/crypto/chelsio/chtls/chtls_cm.h | 202 +++
drivers/crypto/chelsio/chtls/chtls_hw.c | 394 +
drivers/crypto/chelsio/chtls/chtls_io.c | 1867 +
drivers/crypto/chelsio/chtls/chtls_main.c | 600 +++
drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c | 32 +-
drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h |7 +
drivers/net/ethernet/chelsio/cxgb4/sge.c| 98 +-
drivers/net/ethernet/chelsio/cxgb4/t4_msg.h | 121 +-
drivers/net/ethernet/chelsio/cxgb4/t4_regs.h|2 +
drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h | 165 +-
include/linux/netdev_features.h |2 +
include/net/tls.h | 23 +
include/uapi/linux/tls.h|1 +
net/core/ethtool.c |1 +
net/ipv4/tcp_minisocks.c|1 +
net/tls/tls_main.c | 123 +-
23 files changed, 6256 insertions(+), 24 deletions(-)
create mode 100644 drivers/crypto/chelsio/chtls/Makefile
create mode 100644 drivers/crypto/chelsio/chtls/
[Crypto v7 01/12] tls: tls_device struct to register TLS drivers
tls_device structure to register Inline TLS drivers
with net/tls
Signed-off-by: Atul Gupta
---
include/net/tls.h | 23 +++
1 file changed, 23 insertions(+)
diff --git a/include/net/tls.h b/include/net/tls.h
index 4913430..e315bf9 100644
--- a/include/net/tls.h
+++ b/include/net/tls.h
@@ -55,6 +55,27 @@
#define TLS_RECORD_TYPE_DATA 0x17
#define TLS_AAD_SPACE_SIZE 13
+#define TLS_DEVICE_NAME_MAX32
+
+enum {
+ TLS_BASE_TX,
+ TLS_SW_TX,
+ TLS_FULL_HW, /* TLS record processed Inline */
+ TLS_NUM_CONFIG,
+};
+extern struct proto tls_prots[TLS_NUM_CONFIG];
+
+struct tls_device {
+ char name[TLS_DEVICE_NAME_MAX];
+ struct list_head dev_list;
+
+ /* netdev present in registered inline tls driver */
+ int (*netdev)(struct tls_device *device,
+ struct net_device *netdev);
+ int (*feature)(struct tls_device *device);
+ int (*hash)(struct tls_device *device, struct sock *sk);
+ void (*unhash)(struct tls_device *device, struct sock *sk);
+};
struct tls_sw_context {
struct crypto_aead *aead_send;
@@ -256,5 +277,7 @@ static inline struct tls_offload_context *tls_offload_ctx(
int tls_proccess_cmsg(struct sock *sk, struct msghdr *msg,
unsigned char *record_type);
+void tls_register_device(struct tls_device *device);
+void tls_unregister_device(struct tls_device *device);
#endif /* _TLS_OFFLOAD_H */
--
1.8.3.1
[Crypto v7 02/12] ethtool: enable Inline TLS in HW
Signed-off-by: Atul Gupta
---
include/linux/netdev_features.h | 2 ++
net/core/ethtool.c | 1 +
2 files changed, 3 insertions(+)
diff --git a/include/linux/netdev_features.h b/include/linux/netdev_features.h
index db84c51..aacabe2 100644
--- a/include/linux/netdev_features.h
+++ b/include/linux/netdev_features.h
@@ -79,6 +79,7 @@ enum {
NETIF_F_RX_UDP_TUNNEL_PORT_BIT, /* Offload of RX port for UDP tunnels */
NETIF_F_GRO_HW_BIT, /* Hardware Generic receive offload */
+ NETIF_F_HW_TLS_INLINE_BIT, /* Offload TLS record */
/*
* Add your fresh new feature above and remember to update
@@ -145,6 +146,7 @@ enum {
#define NETIF_F_HW_ESP __NETIF_F(HW_ESP)
#define NETIF_F_HW_ESP_TX_CSUM __NETIF_F(HW_ESP_TX_CSUM)
#defineNETIF_F_RX_UDP_TUNNEL_PORT __NETIF_F(RX_UDP_TUNNEL_PORT)
+#define NETIF_F_HW_TLS_INLINE __NETIF_F(HW_TLS_INLINE)
#define for_each_netdev_feature(mask_addr, bit)\
for_each_set_bit(bit, (unsigned long *)mask_addr, NETDEV_FEATURE_COUNT)
diff --git a/net/core/ethtool.c b/net/core/ethtool.c
index 494e6a5..ab16781 100644
--- a/net/core/ethtool.c
+++ b/net/core/ethtool.c
@@ -107,6 +107,7 @@ int ethtool_op_get_ts_info(struct net_device *dev, struct
ethtool_ts_info *info)
[NETIF_F_HW_ESP_BIT] = "esp-hw-offload",
[NETIF_F_HW_ESP_TX_CSUM_BIT] = "esp-tx-csum-hw-offload",
[NETIF_F_RX_UDP_TUNNEL_PORT_BIT] = "rx-udp_tunnel-port-offload",
+ [NETIF_F_HW_TLS_INLINE_BIT] = "tls-inline",
};
static const char
--
1.8.3.1
[Crypto v7 03/12] tls: support for inline tls
Facility to register Inline TLS drivers to net/tls. Setup
TLS_FULL_HW prot to listen on offload device.
Cases handled
1. Inline TLS device exists, setup prot for TLS_FULL_HW
2. Atleast one Inline TLS exists, sets TLS_FULL_HW. If
non-inline capable device establish connection, move to TLS_SW_TX
3. default mode TLS_SW_TX continues
Signed-off-by: Atul Gupta
---
net/tls/tls_main.c | 123 ++---
1 file changed, 116 insertions(+), 7 deletions(-)
diff --git a/net/tls/tls_main.c b/net/tls/tls_main.c
index b0d5fce..34f8781 100644
--- a/net/tls/tls_main.c
+++ b/net/tls/tls_main.c
@@ -38,6 +38,7 @@
#include
#include
#include
+#include
#include
@@ -45,13 +46,9 @@
MODULE_DESCRIPTION("Transport Layer Security Support");
MODULE_LICENSE("Dual BSD/GPL");
-enum {
- TLS_BASE_TX,
- TLS_SW_TX,
- TLS_NUM_CONFIG,
-};
-
-static struct proto tls_prots[TLS_NUM_CONFIG];
+static LIST_HEAD(device_list);
+static DEFINE_MUTEX(device_mutex);
+struct proto tls_prots[TLS_NUM_CONFIG];
static inline void update_sk_prot(struct sock *sk, struct tls_context *ctx)
{
@@ -260,6 +257,37 @@ static void tls_sk_proto_close(struct sock *sk, long
timeout)
sk_proto_close(sk, timeout);
}
+static struct net_device *get_netdev(struct sock *sk)
+{
+ struct inet_sock *inet = inet_sk(sk);
+ struct net_device *netdev;
+
+ netdev = dev_get_by_index(sock_net(sk), inet->cork.fl.flowi_oif);
+ return netdev;
+}
+
+static int tls_offload_dev_absent(struct sock *sk)
+{
+ struct net_device *netdev;
+ struct tls_device *dev;
+ int rc = 0;
+
+ netdev = get_netdev(sk);
+ if (!netdev)
+ return -EINVAL;
+
+ mutex_lock(&device_mutex);
+ list_for_each_entry(dev, &device_list, dev_list) {
+ if (dev->netdev && dev->netdev(dev, netdev)) {
+ rc = -EEXIST;
+ break;
+ }
+ }
+ mutex_unlock(&device_mutex);
+ dev_put(netdev);
+ return rc;
+}
+
static int do_tls_getsockopt_tx(struct sock *sk, char __user *optval,
int __user *optlen)
{
@@ -403,6 +431,15 @@ static int do_tls_setsockopt_tx(struct sock *sk, char
__user *optval,
goto err_crypto_info;
}
+ rc = tls_offload_dev_absent(sk);
+ if (rc == -EINVAL) {
+ goto out;
+ } else if (rc == -EEXIST) {
+ /* Retain HW unhash for cleanup and move to SW Tx */
+ sk->sk_prot[TLS_BASE_TX].unhash =
+ sk->sk_prot[TLS_FULL_HW].unhash;
+ }
+
/* currently SW is default, we will have ethtool in future */
rc = tls_set_sw_offload(sk, ctx);
tx_conf = TLS_SW_TX;
@@ -450,6 +487,54 @@ static int tls_setsockopt(struct sock *sk, int level, int
optname,
return do_tls_setsockopt(sk, optname, optval, optlen);
}
+static int tls_hw_prot(struct sock *sk)
+{
+ struct tls_context *ctx = tls_get_ctx(sk);
+ struct tls_device *dev;
+
+ mutex_lock(&device_mutex);
+ list_for_each_entry(dev, &device_list, dev_list) {
+ if (dev->feature && dev->feature(dev)) {
+ ctx->tx_conf = TLS_FULL_HW;
+ update_sk_prot(sk, ctx);
+ break;
+ }
+ }
+ mutex_unlock(&device_mutex);
+ return ctx->tx_conf;
+}
+
+static void tls_hw_unhash(struct sock *sk)
+{
+ struct tls_device *dev;
+
+ mutex_lock(&device_mutex);
+ list_for_each_entry(dev, &device_list, dev_list) {
+ if (dev->unhash)
+ dev->unhash(dev, sk);
+ }
+ mutex_unlock(&device_mutex);
+ sk->sk_prot->unhash(sk);
+}
+
+static int tls_hw_hash(struct sock *sk)
+{
+ struct tls_device *dev;
+ int err;
+
+ err = sk->sk_prot->hash(sk);
+ mutex_lock(&device_mutex);
+ list_for_each_entry(dev, &device_list, dev_list) {
+ if (dev->hash)
+ err |= dev->hash(dev, sk);
+ }
+ mutex_unlock(&device_mutex);
+
+ if (err)
+ tls_hw_unhash(sk);
+ return err;
+}
+
static int tls_init(struct sock *sk)
{
struct inet_connection_sock *icsk = inet_csk(sk);
@@ -477,6 +562,9 @@ static int tls_init(struct sock *sk)
ctx->sk_proto_close = sk->sk_prot->close;
ctx->tx_conf = TLS_BASE_TX;
+ if (tls_hw_prot(sk) == TLS_FULL_HW)
+ goto out;
+
update_sk_prot(sk, ctx);
out:
return rc;
@@ -500,7 +588,27 @@ static void build_protos(struct proto *prot, struct proto
*base)
prot[TLS_SW_TX] = prot[TLS_BASE_TX];
prot[TLS_SW_TX].sendmsg = tls_sw_sendmsg;
prot[TLS_SW_TX].sendpage
[Crypto v7 04/12] chtls: structure and macro definiton
Inline TLS state, connection management. Supporting macros definition.
Signed-off-by: Atul Gupta
---
drivers/crypto/chelsio/chtls/chtls.h| 487
drivers/crypto/chelsio/chtls/chtls_cm.h | 202 +
2 files changed, 689 insertions(+)
create mode 100644 drivers/crypto/chelsio/chtls/chtls.h
create mode 100644 drivers/crypto/chelsio/chtls/chtls_cm.h
diff --git a/drivers/crypto/chelsio/chtls/chtls.h
b/drivers/crypto/chelsio/chtls/chtls.h
new file mode 100644
index 000..3ae7145
--- /dev/null
+++ b/drivers/crypto/chelsio/chtls/chtls.h
@@ -0,0 +1,487 @@
+/*
+ * Copyright (c) 2016 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef __CHTLS_H__
+#define __CHTLS_H__
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "t4fw_api.h"
+#include "t4_msg.h"
+#include "cxgb4.h"
+#include "cxgb4_uld.h"
+#include "l2t.h"
+#include "chcr_algo.h"
+#include "chcr_core.h"
+#include "chcr_crypto.h"
+
+#define CIPHER_BLOCK_SIZE 16
+#define MAX_IVS_PAGE256
+#define TLS_KEY_CONTEXT_SZ 64
+#define TLS_HEADER_LENGTH 5
+#define SCMD_CIPH_MODE_AES_GCM 2
+#define GCM_TAG_SIZE16
+#define AEAD_EXPLICIT_DATA_SIZE 8
+/* Any MFS size should work and come from openssl */
+#define TLS_MFS16384
+
+#define SOCK_INLINE (31)
+#define RSS_HDR sizeof(struct rss_header)
+
+enum {
+ CHTLS_KEY_CONTEXT_DSGL,
+ CHTLS_KEY_CONTEXT_IMM,
+ CHTLS_KEY_CONTEXT_DDR,
+};
+
+enum {
+ CHTLS_LISTEN_START,
+ CHTLS_LISTEN_STOP,
+};
+
+/* Flags for return value of CPL message handlers */
+enum {
+ CPL_RET_BUF_DONE = 1, /* buffer processing done */
+ CPL_RET_BAD_MSG = 2,/* bad CPL message */
+ CPL_RET_UNKNOWN_TID = 4 /* unexpected unknown TID */
+};
+
+#define TLS_RCV_ST_READ_HEADER 0xF0
+#define TLS_RCV_ST_READ_BODY0xF1
+#define TLS_RCV_ST_READ_DONE0xF2
+#define TLS_RCV_ST_READ_NB 0xF3
+
+#define RSPQ_HASH_BITS 5
+#define LISTEN_INFO_HASH_SIZE 32
+struct listen_info {
+ struct listen_info *next; /* Link to next entry */
+ struct sock *sk; /* The listening socket */
+ unsigned int stid; /* The server TID */
+};
+
+enum {
+ T4_LISTEN_START_PENDING,
+ T4_LISTEN_STARTED
+};
+
+enum csk_flags {
+ CSK_CALLBACKS_CHKD, /* socket callbacks have been sanitized */
+ CSK_ABORT_REQ_RCVD, /* received one ABORT_REQ_RSS message */
+ CSK_TX_MORE_DATA, /* sending ULP data; don't set SHOVE bit */
+ CSK_TX_WAIT_IDLE, /* suspend Tx until in-flight data is ACKed */
+ CSK_ABORT_SHUTDOWN, /* shouldn't send more abort requests */
+ CSK_ABORT_RPL_PENDING, /* expecting an abort reply */
+ CSK_CLOSE_CON_REQUESTED,/* we've sent a close_conn_req */
+ CSK_TX_DATA_SENT, /* sent a TX_DATA WR on this connection */
+ CSK_TX_FAILOVER,/* Tx traffic failing over */
+ CSK_UPDATE_RCV_WND, /* Need to update rcv window */
+ CSK_RST_ABORTED,/* outgoing RST was aborted */
+ CSK_TLS_HANDSHK,/* TLS Handshake */
+};
+
+struct listen_ctx {
+ struct sock *lsk;
+ struct chtls_dev *cdev;
+ u32 state;
+};
+
+struct key_map {
+ unsigned long *addr;
+ unsigned int start;
+ unsigned int available;
+ unsigned int size;
+ spinlock_t lock; /* lock for key id request from map */
+} __packed;
+
+struct tls_scmd {
+ u32 seqno_numivs;
+ u32 ivgen_hdrlen;
+};
+
+struct chtls_dev {
+ struct tls_device tlsdev;
+ struct list_head list;
+ struct cxgb4_lld_info *lldi;
+ struct pci_dev *pdev;
+ struct listen_info *listen_hash_tab[LISTEN_INFO_HASH_SIZE];
+ spinlock_t listen_lock; /* lock for listen list */
+ struct net_device **ports;
+ struct tid_info *tids;
+ unsigned int pfvf;
+ const unsigned short *mtus;
+
+ spinlock_t aidr_lock cacheline_aligned_in_smp;
+ struct idr aidr; /* ATID id space */
+ struct idr hwtid_idr;
+ struct idr stid_idr;
+
+ spinlock_t idr_lock cacheline_aligned_in_smp;
+
+ struct net_device *egr_dev[NCHAN * 2];
+ struct sk_buff *rspq_skb_cache[1 << RSPQ_HASH_BITS];
+ struct sk_buff *askb;
+
+ struct sk_buff_head deferq;
+ struct work_struct deferq_task;
+
+ struct list_head list_node;
+ struct list_head rcu_node;
+ struct list_head na_node;
+
[Crypto v7 08/12] chtls: Key program
Program the tx and rx key on chip.
Signed-off-by: Atul Gupta
---
drivers/crypto/chelsio/chtls/chtls_hw.c | 394
1 file changed, 394 insertions(+)
create mode 100644 drivers/crypto/chelsio/chtls/chtls_hw.c
diff --git a/drivers/crypto/chelsio/chtls/chtls_hw.c
b/drivers/crypto/chelsio/chtls/chtls_hw.c
new file mode 100644
index 000..c3e17159
--- /dev/null
+++ b/drivers/crypto/chelsio/chtls/chtls_hw.c
@@ -0,0 +1,394 @@
+/*
+ * Copyright (c) 2017 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Written by: Atul Gupta (atul.gu...@chelsio.com)
+ */
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "chtls.h"
+#include "chtls_cm.h"
+
+static void __set_tcb_field_direct(struct chtls_sock *csk,
+ struct cpl_set_tcb_field *req, u16 word,
+ u64 mask, u64 val, u8 cookie, int no_reply)
+{
+ struct ulptx_idata *sc;
+
+ INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, csk->tid);
+ req->wr.wr_mid |= htonl(FW_WR_FLOWID_V(csk->tid));
+ req->reply_ctrl = htons(NO_REPLY_V(no_reply) |
+ QUEUENO_V(csk->rss_qid));
+ req->word_cookie = htons(TCB_WORD_V(word) | TCB_COOKIE_V(cookie));
+ req->mask = cpu_to_be64(mask);
+ req->val = cpu_to_be64(val);
+ sc = (struct ulptx_idata *)(req + 1);
+ sc->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP));
+ sc->len = htonl(0);
+}
+
+static void __set_tcb_field(struct sock *sk, struct sk_buff *skb, u16 word,
+ u64 mask, u64 val, u8 cookie, int no_reply)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+ struct cpl_set_tcb_field *req;
+ struct ulptx_idata *sc;
+ unsigned int wrlen = roundup(sizeof(*req) + sizeof(*sc), 16);
+
+ req = (struct cpl_set_tcb_field *)__skb_put(skb, wrlen);
+ __set_tcb_field_direct(csk, req, word, mask, val, cookie, no_reply);
+ set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id);
+}
+
+static int chtls_set_tcb_field(struct sock *sk, u16 word, u64 mask, u64 val)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+ struct sk_buff *skb;
+ struct cpl_set_tcb_field *req;
+ struct ulptx_idata *sc;
+ unsigned int wrlen = roundup(sizeof(*req) + sizeof(*sc), 16);
+ unsigned int credits_needed = DIV_ROUND_UP(wrlen, 16);
+
+ skb = alloc_skb(wrlen, GFP_ATOMIC);
+ if (!skb)
+ return -ENOMEM;
+
+ __set_tcb_field(sk, skb, word, mask, val, 0, 1);
+ set_queue(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA, sk);
+ csk->wr_credits -= credits_needed;
+ csk->wr_unacked += credits_needed;
+ enqueue_wr(csk, skb);
+ cxgb4_ofld_send(csk->egress_dev, skb);
+ return 0;
+}
+
+/*
+ * Set one of the t_flags bits in the TCB.
+ */
+int chtls_set_tcb_tflag(struct sock *sk, unsigned int bit_pos, int val)
+{
+ return chtls_set_tcb_field(sk, 1, 1ULL << bit_pos,
+ val << bit_pos);
+}
+
+static int chtls_set_tcb_keyid(struct sock *sk, int keyid)
+{
+ return chtls_set_tcb_field(sk, 31, 0xULL, keyid);
+}
+
+static int chtls_set_tcb_seqno(struct sock *sk)
+{
+ return chtls_set_tcb_field(sk, 28, ~0ULL, 0);
+}
+
+static int chtls_set_tcb_quiesce(struct sock *sk, int val)
+{
+ return chtls_set_tcb_field(sk, 1, (1ULL << TF_RX_QUIESCE_S),
+ TF_RX_QUIESCE_V(val));
+}
+
+static void *chtls_alloc_mem(unsigned long size)
+{
+ void *p = kmalloc(size, GFP_KERNEL);
+
+ if (!p)
+ p = vmalloc(size);
+ if (p)
+ memset(p, 0, size);
+ return p;
+}
+
+static void chtls_free_mem(void *addr)
+{
+ unsigned long p = (unsigned long)addr;
+
+ if (p >= VMALLOC_START && p < VMALLOC_END)
+ vfree(addr);
+ else
+ kfree(addr);
+}
+
+/* TLS Key bitmap processing */
+int chtls_init_kmap(struct chtls_dev *cdev, struct cxgb4_lld_info *lldi)
+{
+ unsigned int num_key_ctx, bsize;
+
+ num_key_ctx = (lldi->vr->key.size / TLS_KEY_CONTEXT_SZ);
+ bsize = BITS_TO_LONGS(num_key_ctx);
+
+ cdev->kmap.size = num_key_ctx;
+ cdev->kmap.available = bsize;
+ cdev->kmap.addr = chtls_alloc_mem(sizeof(*cdev->kmap.addr) *
+ bsize);
+ if (!cdev->kmap.addr)
+ return -1;
+
+ cdev->kmap.start = lldi->vr->key.start;
+ spin_lock_init(&cdev->kmap.lock);
+ return 0;
+}
+
+void chtls_free_kmap(st
[Crypto v7 12/12] Makefile Kconfig
Entry for Inline TLS as another driver dependent on cxgb4 and chcr Signed-off-by: Atul Gupta --- drivers/crypto/chelsio/Kconfig| 11 +++ drivers/crypto/chelsio/Makefile | 1 + drivers/crypto/chelsio/chtls/Makefile | 4 3 files changed, 16 insertions(+) create mode 100644 drivers/crypto/chelsio/chtls/Makefile diff --git a/drivers/crypto/chelsio/Kconfig b/drivers/crypto/chelsio/Kconfig index 5ae9f87..930d82d 100644 --- a/drivers/crypto/chelsio/Kconfig +++ b/drivers/crypto/chelsio/Kconfig @@ -29,3 +29,14 @@ config CHELSIO_IPSEC_INLINE default n ---help--- Enable support for IPSec Tx Inline. + +config CRYPTO_DEV_CHELSIO_TLS +tristate "Chelsio Crypto Inline TLS Driver" +depends on CHELSIO_T4 +depends on TLS +select CRYPTO_DEV_CHELSIO +---help--- + Support Chelsio Inline TLS with Chelsio crypto accelerator. + + To compile this driver as a module, choose M here: the module + will be called chtls. diff --git a/drivers/crypto/chelsio/Makefile b/drivers/crypto/chelsio/Makefile index eaecaf1..639e571 100644 --- a/drivers/crypto/chelsio/Makefile +++ b/drivers/crypto/chelsio/Makefile @@ -3,3 +3,4 @@ ccflags-y := -Idrivers/net/ethernet/chelsio/cxgb4 obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chcr.o chcr-objs := chcr_core.o chcr_algo.o chcr-$(CONFIG_CHELSIO_IPSEC_INLINE) += chcr_ipsec.o +obj-$(CONFIG_CRYPTO_DEV_CHELSIO_TLS) += chtls/ diff --git a/drivers/crypto/chelsio/chtls/Makefile b/drivers/crypto/chelsio/chtls/Makefile new file mode 100644 index 000..df13795 --- /dev/null +++ b/drivers/crypto/chelsio/chtls/Makefile @@ -0,0 +1,4 @@ +ccflags-y := -Idrivers/net/ethernet/chelsio/cxgb4 -Idrivers/crypto/chelsio/ + +obj-$(CONFIG_CRYPTO_DEV_CHELSIO_TLS) += chtls.o +chtls-objs := chtls_main.o chtls_cm.o chtls_io.o chtls_hw.o -- 1.8.3.1
[Crypto v7 10/12] chtls: Inline crypto request Tx/Rx
TLS handler for record transmit and receive.
Create Inline TLS work request and post to FW.
Signed-off-by: Atul Gupta
---
drivers/crypto/chelsio/chtls/chtls_io.c | 1867 +++
1 file changed, 1867 insertions(+)
create mode 100644 drivers/crypto/chelsio/chtls/chtls_io.c
diff --git a/drivers/crypto/chelsio/chtls/chtls_io.c
b/drivers/crypto/chelsio/chtls/chtls_io.c
new file mode 100644
index 000..0c5d6c1
--- /dev/null
+++ b/drivers/crypto/chelsio/chtls/chtls_io.c
@@ -0,0 +1,1867 @@
+/*
+ * Copyright (c) 2017 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Written by: Atul Gupta (atul.gu...@chelsio.com)
+ */
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "chtls.h"
+#include "chtls_cm.h"
+
+static bool is_tls_hw(struct chtls_sock *csk)
+{
+ return csk->tlshws.ofld;
+}
+
+static bool is_tls_rx(struct chtls_sock *csk)
+{
+ return (csk->tlshws.rxkey >= 0);
+}
+
+static bool is_tls_tx(struct chtls_sock *csk)
+{
+ return (csk->tlshws.txkey >= 0);
+}
+
+static bool is_tls_skb(struct chtls_sock *csk, const struct sk_buff *skb)
+{
+ return (is_tls_hw(csk) && skb_ulp_tls_skb_flags(skb));
+}
+
+static int key_size(void *sk)
+{
+ return 16; /* Key on DDR */
+}
+
+#define ceil(x, y) \
+ ({ unsigned long __x = (x), __y = (y); (__x + __y - 1) / __y; })
+
+static int data_sgl_len(const struct sk_buff *skb)
+{
+ unsigned int cnt;
+
+ cnt = skb_shinfo(skb)->nr_frags;
+ return (sgl_len(cnt) * 8);
+}
+
+static int nos_ivs(struct sock *sk, unsigned int size)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+
+ return ceil(size, csk->tlshws.mfs);
+}
+
+#define TLS_WR_CPL_LEN \
+ (sizeof(struct fw_tlstx_data_wr) + \
+ sizeof(struct cpl_tx_tls_sfo))
+
+static int is_ivs_imm(struct sock *sk, const struct sk_buff *skb)
+{
+ int ivs_size = nos_ivs(sk, skb->len) * CIPHER_BLOCK_SIZE;
+ int hlen = TLS_WR_CPL_LEN + data_sgl_len(skb);
+
+ if ((hlen + key_size(sk) + ivs_size) <
+ MAX_IMM_OFLD_TX_DATA_WR_LEN) {
+ ULP_SKB_CB(skb)->ulp.tls.iv = 1;
+ return 1;
+ }
+ ULP_SKB_CB(skb)->ulp.tls.iv = 0;
+ return 0;
+}
+
+static int max_ivs_size(struct sock *sk, int size)
+{
+ return (nos_ivs(sk, size) * CIPHER_BLOCK_SIZE);
+}
+
+static int ivs_size(struct sock *sk, const struct sk_buff *skb)
+{
+ return (is_ivs_imm(sk, skb) ? (nos_ivs(sk, skb->len) *
+CIPHER_BLOCK_SIZE) : 0);
+}
+
+static int flowc_wr_credits(int nparams, int *flowclenp)
+{
+ int flowclen16, flowclen;
+
+ flowclen = offsetof(struct fw_flowc_wr, mnemval[nparams]);
+ flowclen16 = DIV_ROUND_UP(flowclen, 16);
+ flowclen = flowclen16 * 16;
+
+ if (flowclenp)
+ *flowclenp = flowclen;
+
+ return flowclen16;
+}
+
+static struct sk_buff *create_flowc_wr_skb(struct sock *sk,
+ struct fw_flowc_wr *flowc,
+ int flowclen)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+ struct sk_buff *skb;
+
+ skb = alloc_skb(flowclen, GFP_ATOMIC);
+ if (!skb)
+ return NULL;
+
+ memcpy(__skb_put(skb, flowclen), flowc, flowclen);
+ set_queue(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA, sk);
+
+ return skb;
+}
+
+static int send_flowc_wr(struct sock *sk, struct fw_flowc_wr *flowc,
+int flowclen)
+{
+ struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
+ bool syn_sent = (sk->sk_state == TCP_SYN_SENT);
+ struct tcp_sock *tp = tcp_sk(sk);
+ int flowclen16 = flowclen / 16;
+ struct sk_buff *skb;
+
+ if (csk_flag(sk, CSK_TX_DATA_SENT)) {
+ skb = create_flowc_wr_skb(sk, flowc, flowclen);
+ if (!skb)
+ return -ENOMEM;
+
+ if (syn_sent)
+ __skb_queue_tail(&csk->ooo_queue, skb);
+ else
+ skb_entail(sk, skb,
+ ULPCB_FLAG_NO_HDR | ULPCB_FLAG_NO_APPEND);
+ return 0;
+ }
+
+ if (!syn_sent) {
+ int ret;
+
+ ret = cxgb4_immdata_send(csk->egress_dev,
+csk->txq_idx,
+flowc, flowclen);
+ if (!ret)
+ return flowclen16;
+ }
+ skb = create_flowc_wr_skb(sk, flowc, flowclen);
+ if (!skb)
+ return -
