1Overview
2========
3
4Transport Layer Security (TLS) is a Upper Layer Protocol (ULP) that runs over
5TCP. TLS provides end-to-end data integrity and confidentiality.
6
7User interface
8==============
9
10Creating a TLS connection
11-------------------------
12
13First create a new TCP socket and set the TLS ULP.
14
15  sock = socket(AF_INET, SOCK_STREAM, 0);
16  setsockopt(sock, SOL_TCP, TCP_ULP, "tls", sizeof("tls"));
17
18Setting the TLS ULP allows us to set/get TLS socket options. Currently
19only the symmetric encryption is handled in the kernel.  After the TLS
20handshake is complete, we have all the parameters required to move the
21data-path to the kernel. There is a separate socket option for moving
22the transmit and the receive into the kernel.
23
24  /* From linux/tls.h */
25  struct tls_crypto_info {
26          unsigned short version;
27          unsigned short cipher_type;
28  };
29
30  struct tls12_crypto_info_aes_gcm_128 {
31          struct tls_crypto_info info;
32          unsigned char iv[TLS_CIPHER_AES_GCM_128_IV_SIZE];
33          unsigned char key[TLS_CIPHER_AES_GCM_128_KEY_SIZE];
34          unsigned char salt[TLS_CIPHER_AES_GCM_128_SALT_SIZE];
35          unsigned char rec_seq[TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE];
36  };
37
38
39  struct tls12_crypto_info_aes_gcm_128 crypto_info;
40
41  crypto_info.info.version = TLS_1_2_VERSION;
42  crypto_info.info.cipher_type = TLS_CIPHER_AES_GCM_128;
43  memcpy(crypto_info.iv, iv_write, TLS_CIPHER_AES_GCM_128_IV_SIZE);
44  memcpy(crypto_info.rec_seq, seq_number_write,
45					TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE);
46  memcpy(crypto_info.key, cipher_key_write, TLS_CIPHER_AES_GCM_128_KEY_SIZE);
47  memcpy(crypto_info.salt, implicit_iv_write, TLS_CIPHER_AES_GCM_128_SALT_SIZE);
48
49  setsockopt(sock, SOL_TLS, TLS_TX, &crypto_info, sizeof(crypto_info));
50
51Transmit and receive are set separately, but the setup is the same, using either
52TLS_TX or TLS_RX.
53
54Sending TLS application data
55----------------------------
56
57After setting the TLS_TX socket option all application data sent over this
58socket is encrypted using TLS and the parameters provided in the socket option.
59For example, we can send an encrypted hello world record as follows:
60
61  const char *msg = "hello world\n";
62  send(sock, msg, strlen(msg));
63
64send() data is directly encrypted from the userspace buffer provided
65to the encrypted kernel send buffer if possible.
66
67The sendfile system call will send the file's data over TLS records of maximum
68length (2^14).
69
70  file = open(filename, O_RDONLY);
71  fstat(file, &stat);
72  sendfile(sock, file, &offset, stat.st_size);
73
74TLS records are created and sent after each send() call, unless
75MSG_MORE is passed.  MSG_MORE will delay creation of a record until
76MSG_MORE is not passed, or the maximum record size is reached.
77
78The kernel will need to allocate a buffer for the encrypted data.
79This buffer is allocated at the time send() is called, such that
80either the entire send() call will return -ENOMEM (or block waiting
81for memory), or the encryption will always succeed.  If send() returns
82-ENOMEM and some data was left on the socket buffer from a previous
83call using MSG_MORE, the MSG_MORE data is left on the socket buffer.
84
85Receiving TLS application data
86------------------------------
87
88After setting the TLS_RX socket option, all recv family socket calls
89are decrypted using TLS parameters provided.  A full TLS record must
90be received before decryption can happen.
91
92  char buffer[16384];
93  recv(sock, buffer, 16384);
94
95Received data is decrypted directly in to the user buffer if it is
96large enough, and no additional allocations occur.  If the userspace
97buffer is too small, data is decrypted in the kernel and copied to
98userspace.
99
100EINVAL is returned if the TLS version in the received message does not
101match the version passed in setsockopt.
102
103EMSGSIZE is returned if the received message is too big.
104
105EBADMSG is returned if decryption failed for any other reason.
106
107Send TLS control messages
108-------------------------
109
110Other than application data, TLS has control messages such as alert
111messages (record type 21) and handshake messages (record type 22), etc.
112These messages can be sent over the socket by providing the TLS record type
113via a CMSG. For example the following function sends @data of @length bytes
114using a record of type @record_type.
115
116/* send TLS control message using record_type */
117  static int klts_send_ctrl_message(int sock, unsigned char record_type,
118                                  void *data, size_t length)
119  {
120        struct msghdr msg = {0};
121        int cmsg_len = sizeof(record_type);
122        struct cmsghdr *cmsg;
123        char buf[CMSG_SPACE(cmsg_len)];
124        struct iovec msg_iov;   /* Vector of data to send/receive into.  */
125
126        msg.msg_control = buf;
127        msg.msg_controllen = sizeof(buf);
128        cmsg = CMSG_FIRSTHDR(&msg);
129        cmsg->cmsg_level = SOL_TLS;
130        cmsg->cmsg_type = TLS_SET_RECORD_TYPE;
131        cmsg->cmsg_len = CMSG_LEN(cmsg_len);
132        *CMSG_DATA(cmsg) = record_type;
133        msg.msg_controllen = cmsg->cmsg_len;
134
135        msg_iov.iov_base = data;
136        msg_iov.iov_len = length;
137        msg.msg_iov = &msg_iov;
138        msg.msg_iovlen = 1;
139
140        return sendmsg(sock, &msg, 0);
141  }
142
143Control message data should be provided unencrypted, and will be
144encrypted by the kernel.
145
146Receiving TLS control messages
147------------------------------
148
149TLS control messages are passed in the userspace buffer, with message
150type passed via cmsg.  If no cmsg buffer is provided, an error is
151returned if a control message is received.  Data messages may be
152received without a cmsg buffer set.
153
154  char buffer[16384];
155  char cmsg[CMSG_SPACE(sizeof(unsigned char))];
156  struct msghdr msg = {0};
157  msg.msg_control = cmsg;
158  msg.msg_controllen = sizeof(cmsg);
159
160  struct iovec msg_iov;
161  msg_iov.iov_base = buffer;
162  msg_iov.iov_len = 16384;
163
164  msg.msg_iov = &msg_iov;
165  msg.msg_iovlen = 1;
166
167  int ret = recvmsg(sock, &msg, 0 /* flags */);
168
169  struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg);
170  if (cmsg->cmsg_level == SOL_TLS &&
171      cmsg->cmsg_type == TLS_GET_RECORD_TYPE) {
172      int record_type = *((unsigned char *)CMSG_DATA(cmsg));
173      // Do something with record_type, and control message data in
174      // buffer.
175      //
176      // Note that record_type may be == to application data (23).
177  } else {
178      // Buffer contains application data.
179  }
180
181recv will never return data from mixed types of TLS records.
182
183Integrating in to userspace TLS library
184---------------------------------------
185
186At a high level, the kernel TLS ULP is a replacement for the record
187layer of a userspace TLS library.
188
189A patchset to OpenSSL to use ktls as the record layer is here:
190
191https://github.com/Mellanox/openssl/commits/tls_rx2
192
193An example of calling send directly after a handshake using
194gnutls.  Since it doesn't implement a full record layer, control
195messages are not supported:
196
197https://github.com/ktls/af_ktls-tool/commits/RX
198