Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0-or-later
2 : /* QUIC kernel implementation
3 : * (C) Copyright Red Hat Corp. 2023
4 : *
5 : * This file is part of the QUIC kernel implementation
6 : *
7 : * Initialization/cleanup for QUIC protocol support.
8 : *
9 : * Written or modified by:
10 : * Xin Long <lucien.xin@gmail.com>
11 : */
12 :
13 : #include <net/inet_common.h>
14 : #include <linux/version.h>
15 : #include <asm/ioctls.h>
16 : #include <net/tls.h>
17 :
18 : #include "socket.h"
19 :
20 : static DEFINE_PER_CPU(int, quic_memory_per_cpu_fw_alloc);
21 : static unsigned long quic_memory_pressure;
22 : static atomic_long_t quic_memory_allocated;
23 :
24 0 : static void quic_enter_memory_pressure(struct sock *sk)
25 : {
26 0 : WRITE_ONCE(quic_memory_pressure, 1);
27 0 : }
28 :
29 : /* Check if a matching request sock already exists. Match is based on source/destination
30 : * addresses and DCID.
31 : */
32 743 : bool quic_request_sock_exists(struct sock *sk)
33 : {
34 743 : struct quic_packet *packet = quic_packet(sk);
35 743 : struct quic_request_sock *req;
36 :
37 743 : list_for_each_entry(req, quic_reqs(sk), list) {
38 132 : if (!memcmp(&req->saddr, &packet->saddr, sizeof(req->saddr)) &&
39 198 : !memcmp(&req->daddr, &packet->daddr, sizeof(req->daddr)) &&
40 66 : !quic_conn_id_cmp(&req->dcid, &packet->dcid))
41 : return true;
42 : }
43 : return false;
44 : }
45 :
46 : /* Create and enqueue a QUIC request sock for a new incoming connection. */
47 420 : int quic_request_sock_enqueue(struct sock *sk, struct quic_conn_id *odcid, u8 retry)
48 : {
49 420 : struct quic_packet *packet = quic_packet(sk);
50 420 : struct quic_request_sock *req;
51 :
52 420 : if (sk_acceptq_is_full(sk)) /* Refuse new request if the accept queue is full. */
53 : return -ENOMEM;
54 :
55 420 : req = kzalloc(sizeof(*req), GFP_ATOMIC);
56 420 : if (!req)
57 : return -ENOMEM;
58 :
59 420 : req->version = packet->version;
60 420 : req->daddr = packet->daddr;
61 420 : req->saddr = packet->saddr;
62 420 : req->scid = packet->scid;
63 420 : req->dcid = packet->dcid;
64 420 : req->orig_dcid = *odcid;
65 420 : req->retry = retry;
66 :
67 : /* Enqueue request into the listen socket’s pending list for accept(). */
68 420 : list_add_tail(&req->list, quic_reqs(sk));
69 420 : sk_acceptq_added(sk);
70 420 : return 0;
71 : }
72 :
73 420 : static struct quic_request_sock *quic_request_sock_dequeue(struct sock *sk)
74 : {
75 420 : struct quic_request_sock *req;
76 :
77 420 : req = list_first_entry(quic_reqs(sk), struct quic_request_sock, list);
78 :
79 420 : list_del_init(&req->list);
80 420 : sk_acceptq_removed(sk);
81 420 : return req;
82 : }
83 :
84 : /* Check if a matching accept socket exists. This is needed because an accept socket
85 : * might have been created after this packet was enqueued in the listen socket's backlog.
86 : */
87 677 : bool quic_accept_sock_exists(struct sock *sk, struct sk_buff *skb)
88 : {
89 677 : struct quic_pnspace *space = quic_pnspace(sk, QUIC_CRYPTO_INITIAL);
90 677 : struct quic_packet *packet = quic_packet(sk);
91 677 : bool exist = false;
92 :
93 : /* Skip if packet is newer than the last accept socket creation time. No matching
94 : * socket could exist in this case.
95 : */
96 677 : if (QUIC_SKB_CB(skb)->time > space->time)
97 : return exist;
98 :
99 : /* Look up an accepted socket that matches the packet's addresses and DCID. */
100 19 : local_bh_disable();
101 19 : sk = quic_sock_lookup(skb, &packet->saddr, &packet->daddr, &packet->dcid);
102 19 : if (!sk)
103 0 : goto out;
104 :
105 : /* Found a matching accept socket. Process the packet with this socket. */
106 19 : bh_lock_sock(sk);
107 19 : if (sock_owned_by_user(sk)) {
108 : /* Socket is busy (owned by user context): queue to backlog. */
109 0 : if (sk_add_backlog(sk, skb, READ_ONCE(sk->sk_rcvbuf)))
110 0 : kfree_skb(skb);
111 : } else {
112 : /* Socket not busy: process immediately. */
113 19 : sk->sk_backlog_rcv(sk, skb); /* quic_packet_process(). */
114 : }
115 19 : bh_unlock_sock(sk);
116 19 : exist = true;
117 19 : out:
118 19 : local_bh_enable();
119 19 : return exist;
120 : }
121 :
122 : /* Lookup a connected QUIC socket based on address and dest connection ID.
123 : *
124 : * This function searches the established (non-listening) QUIC socket table for a socket that
125 : * matches the source and dest addresses and, optionally, the dest connection ID (DCID). The
126 : * value returned by quic_path_orig_dcid() might be the original dest connection ID from the
127 : * ClientHello or the Source Connection ID from a Retry packet before.
128 : *
129 : * The DCID is provided from a handshake packet when searching by source connection ID fails,
130 : * such as when the peer has not yet received server's response and updated the DCID.
131 : *
132 : * Return: A pointer to the matching connected socket, or NULL if no match is found.
133 : */
134 4360 : struct sock *quic_sock_lookup(struct sk_buff *skb, union quic_addr *sa, union quic_addr *da,
135 : struct quic_conn_id *dcid)
136 : {
137 4360 : struct net *net = dev_net(skb->dev);
138 4360 : struct quic_path_group *paths;
139 4360 : struct quic_hash_head *head;
140 4360 : struct sock *sk;
141 :
142 4360 : head = quic_sock_head(net, sa, da);
143 4360 : spin_lock(&head->s_lock);
144 8928 : sk_for_each(sk, &head->head) {
145 3618 : if (net != sock_net(sk))
146 1 : continue;
147 3617 : paths = quic_paths(sk);
148 7232 : if (quic_cmp_sk_addr(sk, quic_path_saddr(paths, 0), sa) &&
149 7197 : quic_cmp_sk_addr(sk, quic_path_daddr(paths, 0), da) &&
150 340 : (!dcid || !quic_conn_id_cmp(quic_path_orig_dcid(paths), dcid)))
151 : break;
152 : }
153 4360 : spin_unlock(&head->s_lock);
154 :
155 4360 : return sk;
156 : }
157 :
158 : /* Find the listening QUIC socket for an incoming packet.
159 : *
160 : * This function searches the QUIC socket table for a listening socket that matches the dest
161 : * address and port, and the ALPN(s) if presented in the ClientHello. If multiple listening
162 : * sockets are bound to the same address, port, and ALPN(s) (e.g., via SO_REUSEPORT), this
163 : * function selects a socket from the reuseport group.
164 : *
165 : * Return: A pointer to the matching listening socket, or NULL if no match is found.
166 : */
167 1096 : struct sock *quic_listen_sock_lookup(struct sk_buff *skb, union quic_addr *sa, union quic_addr *da,
168 : struct quic_data *alpns)
169 : {
170 1096 : struct net *net = dev_net(skb->dev);
171 1096 : struct sock *sk = NULL, *tmp;
172 1096 : struct quic_hash_head *head;
173 1096 : struct quic_data alpn;
174 1096 : union quic_addr *a;
175 1096 : u64 length;
176 1096 : u32 len;
177 1096 : u8 *p;
178 :
179 1096 : head = quic_listen_sock_head(net, ntohs(sa->v4.sin_port));
180 1096 : spin_lock(&head->s_lock);
181 :
182 1096 : if (!alpns->len) { /* No ALPN entries present or failed to parse the ALPNs. */
183 2620 : sk_for_each(tmp, &head->head) {
184 : /* If alpns->data != NULL, TLS parsing succeeded but no ALPN was found.
185 : * In this case, only match sockets that have no ALPN set.
186 : */
187 576 : a = quic_path_saddr(quic_paths(tmp), 0);
188 576 : if (net == sock_net(tmp) && quic_cmp_sk_addr(tmp, a, sa) &&
189 576 : (!alpns->data || !quic_alpn(tmp)->len)) {
190 576 : sk = tmp;
191 576 : if (!quic_is_any_addr(a)) /* Prefer specific address match. */
192 : break;
193 : }
194 : }
195 734 : goto unlock;
196 : }
197 :
198 : /* ALPN present: loop through each ALPN entry. */
199 362 : for (p = alpns->data, len = alpns->len; len; len -= length, p += length) {
200 362 : quic_get_int(&p, &len, &length, 1);
201 362 : quic_data(&alpn, p, length);
202 1428 : sk_for_each(tmp, &head->head) {
203 362 : a = quic_path_saddr(quic_paths(tmp), 0);
204 724 : if (net == sock_net(tmp) && quic_cmp_sk_addr(tmp, a, sa) &&
205 362 : quic_data_has(quic_alpn(tmp), &alpn)) {
206 362 : sk = tmp;
207 362 : if (!quic_is_any_addr(a))
208 : break;
209 : }
210 : }
211 362 : if (sk)
212 : break;
213 : }
214 362 : unlock:
215 1096 : spin_unlock(&head->s_lock);
216 :
217 1096 : if (sk && sk->sk_reuseport)
218 0 : sk = reuseport_select_sock(sk, quic_shash(net, da), skb, 1);
219 1096 : return sk;
220 : }
221 :
222 6392452 : static void quic_write_space(struct sock *sk)
223 : {
224 6392452 : struct socket_wq *wq;
225 :
226 6392452 : rcu_read_lock();
227 6392452 : wq = rcu_dereference(sk->sk_wq);
228 12784904 : if (skwq_has_sleeper(wq))
229 26503 : wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT | EPOLLWRNORM | EPOLLWRBAND);
230 6392452 : rcu_read_unlock();
231 6392452 : }
232 :
233 : /* Apply QUIC transport parameters to subcomponents of the socket. */
234 3172 : static void quic_sock_apply_transport_param(struct sock *sk, struct quic_transport_param *p)
235 : {
236 3172 : struct quic_conn_id_set *id_set = p->remote ? quic_source(sk) : quic_dest(sk);
237 :
238 3172 : quic_inq_set_param(sk, p);
239 3172 : quic_outq_set_param(sk, p);
240 3172 : quic_conn_id_set_param(id_set, p);
241 3172 : quic_path_set_param(quic_paths(sk), p);
242 3172 : quic_stream_set_param(quic_streams(sk), p, quic_is_serv(sk));
243 3172 : }
244 :
245 : /* Fetch QUIC transport parameters from subcomponents of the socket. */
246 2079 : static void quic_sock_fetch_transport_param(struct sock *sk, struct quic_transport_param *p)
247 : {
248 2079 : struct quic_conn_id_set *id_set = p->remote ? quic_source(sk) : quic_dest(sk);
249 :
250 2079 : quic_inq_get_param(sk, p);
251 2079 : quic_outq_get_param(sk, p);
252 2079 : quic_conn_id_get_param(id_set, p);
253 2079 : quic_path_get_param(quic_paths(sk), p);
254 2079 : quic_stream_get_param(quic_streams(sk), p, quic_is_serv(sk));
255 2079 : }
256 :
257 0 : static int quic_ioctl(struct sock *sk, int cmd, int *karg)
258 : {
259 0 : int err = 0;
260 :
261 0 : lock_sock(sk);
262 :
263 0 : if (quic_is_listen(sk)) {
264 0 : err = -EINVAL;
265 0 : goto out;
266 : }
267 :
268 0 : switch (cmd) {
269 : case SIOCINQ:
270 0 : *karg = sk_rmem_alloc_get(sk);
271 0 : break;
272 : case SIOCOUTQ:
273 0 : *karg = sk_wmem_alloc_get(sk);
274 0 : break;
275 : case SIOCOUTQNSD:
276 0 : *karg = quic_outq(sk)->unsent_bytes;
277 0 : break;
278 : default:
279 : err = -ENOIOCTLCMD;
280 : break;
281 : }
282 0 : out:
283 0 : release_sock(sk);
284 0 : return err;
285 : }
286 :
287 1119 : static int quic_init_sock(struct sock *sk)
288 : {
289 1119 : struct quic_transport_param *p = &quic_default_param;
290 1119 : u8 i;
291 :
292 1119 : sk->sk_destruct = inet_sock_destruct;
293 1119 : sk->sk_write_space = quic_write_space;
294 1119 : sock_set_flag(sk, SOCK_USE_WRITE_QUEUE);
295 :
296 1119 : quic_conn_id_set_init(quic_source(sk), 1);
297 1119 : quic_conn_id_set_init(quic_dest(sk), 0);
298 1119 : quic_cong_init(quic_cong(sk));
299 :
300 1119 : quic_sock_apply_transport_param(sk, p);
301 :
302 1119 : quic_outq_init(sk);
303 1119 : quic_inq_init(sk);
304 1119 : quic_timer_init(sk);
305 1119 : quic_packet_init(sk);
306 :
307 1119 : if (quic_stream_init(quic_streams(sk)))
308 : return -ENOMEM;
309 :
310 4476 : for (i = 0; i < QUIC_PNSPACE_MAX; i++) {
311 3357 : if (quic_pnspace_init(quic_pnspace(sk, i)))
312 : return -ENOMEM;
313 : }
314 :
315 1119 : WRITE_ONCE(sk->sk_sndbuf, READ_ONCE(sysctl_quic_wmem[1]));
316 1119 : WRITE_ONCE(sk->sk_rcvbuf, READ_ONCE(sysctl_quic_rmem[1]));
317 :
318 1119 : local_bh_disable();
319 1119 : sk_sockets_allocated_inc(sk);
320 1119 : sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
321 1119 : local_bh_enable();
322 :
323 1119 : return 0;
324 : }
325 :
326 1118 : static void quic_destroy_sock(struct sock *sk)
327 : {
328 1118 : u8 i;
329 :
330 1118 : quic_outq_free(sk);
331 1118 : quic_inq_free(sk);
332 1118 : quic_timer_free(sk);
333 :
334 5590 : for (i = 0; i < QUIC_PNSPACE_MAX; i++)
335 3354 : quic_pnspace_free(quic_pnspace(sk, i));
336 5590 : for (i = 0; i < QUIC_CRYPTO_MAX; i++)
337 4472 : quic_crypto_free(quic_crypto(sk, i));
338 :
339 1118 : quic_path_free(sk, quic_paths(sk), 0);
340 1118 : quic_path_free(sk, quic_paths(sk), 1);
341 :
342 1118 : quic_conn_id_set_free(quic_source(sk));
343 1118 : quic_conn_id_set_free(quic_dest(sk));
344 :
345 1118 : quic_stream_free(quic_streams(sk));
346 :
347 1118 : quic_data_free(quic_ticket(sk));
348 1118 : quic_data_free(quic_token(sk));
349 1118 : quic_data_free(quic_alpn(sk));
350 :
351 1118 : local_bh_disable();
352 1118 : sk_sockets_allocated_dec(sk);
353 1118 : sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
354 1118 : local_bh_enable();
355 1118 : }
356 :
357 538 : static int quic_bind(struct sock *sk, struct sockaddr *addr, int addr_len)
358 : {
359 538 : struct quic_path_group *paths = quic_paths(sk);
360 538 : union quic_addr a;
361 538 : int err = -EINVAL;
362 :
363 538 : lock_sock(sk);
364 :
365 538 : if (quic_path_saddr(paths, 0)->v4.sin_port || quic_get_user_addr(sk, &a, addr, addr_len))
366 0 : goto out;
367 :
368 538 : quic_path_set_saddr(paths, 0, &a);
369 538 : err = quic_path_bind(sk, paths, 0);
370 538 : if (err)
371 0 : goto out;
372 538 : quic_set_sk_addr(sk, &a, true);
373 :
374 538 : out:
375 538 : release_sock(sk);
376 538 : return err;
377 : }
378 :
379 581 : static int quic_connect(struct sock *sk, struct sockaddr *addr, int addr_len)
380 : {
381 581 : struct quic_conn_id_set *dest = quic_dest(sk), *source = quic_source(sk);
382 581 : struct quic_crypto *crypto = quic_crypto(sk, QUIC_CRYPTO_INITIAL);
383 581 : struct quic_path_group *paths = quic_paths(sk);
384 581 : struct quic_packet *packet = quic_packet(sk);
385 581 : struct quic_inqueue *inq = quic_inq(sk);
386 581 : struct quic_conn_id conn_id, *active;
387 581 : union quic_addr *sa, a;
388 581 : int err = -EINVAL;
389 :
390 581 : lock_sock(sk);
391 581 : if (!quic_is_closed(sk) || quic_get_user_addr(sk, &a, addr, addr_len))
392 0 : goto out;
393 :
394 : /* Set destination address and resolve route (may also auto-set source address). */
395 581 : quic_path_set_daddr(paths, 0, &a);
396 581 : err = quic_packet_route(sk);
397 581 : if (err < 0)
398 0 : goto out;
399 581 : quic_set_sk_addr(sk, &a, false);
400 :
401 581 : sa = quic_path_saddr(paths, 0);
402 581 : if (!sa->v4.sin_port) { /* Auto-bind if not already bound. */
403 581 : err = quic_path_bind(sk, paths, 0);
404 581 : if (err)
405 0 : goto out;
406 581 : quic_set_sk_addr(sk, sa, true);
407 : }
408 :
409 : /* Generate and add destination and source connection IDs. */
410 581 : quic_conn_id_generate(&conn_id);
411 581 : err = quic_conn_id_add(dest, &conn_id, 0, NULL);
412 581 : if (err)
413 0 : goto out;
414 : /* Save original DCID for validating server's transport parameters. */
415 581 : paths->orig_dcid = conn_id;
416 581 : quic_conn_id_generate(&conn_id);
417 581 : err = quic_conn_id_add(source, &conn_id, 0, sk);
418 581 : if (err)
419 0 : goto free;
420 581 : active = quic_conn_id_active(dest);
421 :
422 : /* Install initial encryption keys for handshake. */
423 581 : err = quic_crypto_set_cipher(crypto, TLS_CIPHER_AES_GCM_128, CRYPTO_ALG_ASYNC);
424 581 : if (err)
425 0 : goto free;
426 581 : err = quic_crypto_initial_keys_install(crypto, active, packet->version, 0);
427 581 : if (err)
428 0 : goto free;
429 :
430 : /* Add socket to hash table, change state to ESTABLISHING, and start idle timer. */
431 581 : err = sk->sk_prot->hash(sk);
432 581 : if (err)
433 0 : goto free;
434 581 : quic_set_state(sk, QUIC_SS_ESTABLISHING);
435 581 : quic_timer_start(sk, QUIC_TIMER_IDLE, inq->timeout);
436 581 : out:
437 581 : release_sock(sk);
438 581 : return err;
439 0 : free:
440 0 : quic_conn_id_set_free(source);
441 0 : quic_conn_id_set_free(dest);
442 0 : quic_crypto_free(crypto);
443 0 : goto out;
444 : }
445 :
446 1119 : static int quic_hash(struct sock *sk)
447 : {
448 1119 : struct quic_path_group *paths = quic_paths(sk);
449 1119 : struct quic_data *alpns = quic_alpn(sk);
450 1119 : struct net *net = sock_net(sk);
451 1119 : struct quic_hash_head *head;
452 1119 : union quic_addr *sa, *da;
453 1119 : struct sock *nsk;
454 1119 : int err = 0, any;
455 :
456 1119 : sa = quic_path_saddr(paths, 0);
457 1119 : da = quic_path_daddr(paths, 0);
458 1119 : if (!sk->sk_max_ack_backlog) { /* Hash a regular socket with source and dest addrs/ports. */
459 1001 : head = quic_sock_head(net, sa, da);
460 1001 : spin_lock_bh(&head->s_lock);
461 1001 : __sk_add_node(sk, &head->head);
462 1001 : spin_unlock_bh(&head->s_lock);
463 1001 : return 0;
464 : }
465 :
466 : /* Hash a listen socket with source port only. */
467 118 : head = quic_listen_sock_head(net, ntohs(sa->v4.sin_port));
468 118 : spin_lock_bh(&head->s_lock);
469 :
470 118 : any = quic_is_any_addr(sa);
471 236 : sk_for_each(nsk, &head->head) {
472 0 : if (net == sock_net(nsk) && quic_cmp_sk_addr(nsk, quic_path_saddr(paths, 0), sa)) {
473 : /* Take the ALPNs into account, which allows directing the request to
474 : * different listening sockets based on the ALPNs.
475 : */
476 0 : if (!quic_data_cmp(alpns, quic_alpn(nsk))) {
477 0 : err = -EADDRINUSE;
478 0 : if (sk->sk_reuseport && nsk->sk_reuseport) {
479 : /* Support SO_REUSEPORT: allow multiple sockets with
480 : * same addr/port/ALPNs.
481 : */
482 0 : err = reuseport_add_sock(sk, nsk, any);
483 0 : if (!err) {
484 0 : __sk_add_node(sk, &head->head);
485 0 : INIT_LIST_HEAD(quic_reqs(sk));
486 : }
487 : }
488 0 : goto out;
489 : }
490 : /* If ALPNs partially match, also consider address in use. */
491 0 : if (quic_data_match(alpns, quic_alpn(nsk))) {
492 0 : err = -EADDRINUSE;
493 0 : goto out;
494 : }
495 : }
496 : }
497 :
498 118 : if (sk->sk_reuseport) { /* If socket uses reuseport, allocate reuseport group. */
499 0 : err = reuseport_alloc(sk, any);
500 0 : if (err)
501 0 : goto out;
502 : }
503 118 : __sk_add_node(sk, &head->head);
504 118 : INIT_LIST_HEAD(quic_reqs(sk));
505 118 : out:
506 118 : spin_unlock_bh(&head->s_lock);
507 118 : return err;
508 : }
509 :
510 2236 : static void quic_unhash(struct sock *sk)
511 : {
512 2236 : struct quic_path_group *paths = quic_paths(sk);
513 2236 : struct quic_request_sock *req, *tmp;
514 2236 : struct net *net = sock_net(sk);
515 2236 : struct quic_hash_head *head;
516 2236 : union quic_addr *sa, *da;
517 :
518 2236 : if (sk_unhashed(sk))
519 : return;
520 :
521 1118 : sa = quic_path_saddr(paths, 0);
522 1118 : da = quic_path_daddr(paths, 0);
523 1118 : if (sk->sk_max_ack_backlog) {
524 : /* Unhash a listen socket: clean up all pending connection requests. */
525 117 : list_for_each_entry_safe(req, tmp, quic_reqs(sk), list) {
526 0 : list_del(&req->list);
527 0 : kfree(req);
528 : }
529 117 : head = quic_listen_sock_head(net, ntohs(sa->v4.sin_port));
530 117 : goto out;
531 : }
532 1001 : head = quic_sock_head(net, sa, da);
533 :
534 1118 : out:
535 1118 : spin_lock_bh(&head->s_lock);
536 1118 : if (rcu_access_pointer(sk->sk_reuseport_cb))
537 0 : reuseport_detach_sock(sk); /* If socket was part of a reuseport group, detach it. */
538 1118 : __sk_del_node_init(sk);
539 1118 : spin_unlock_bh(&head->s_lock);
540 : }
541 :
542 : #define QUIC_MSG_STREAM_FLAGS \
543 : (MSG_STREAM_NEW | MSG_STREAM_FIN | MSG_STREAM_UNI | MSG_STREAM_DONTWAIT)
544 :
545 : #define QUIC_MSG_FLAGS \
546 : (QUIC_MSG_STREAM_FLAGS | MSG_BATCH | MSG_MORE | MSG_DONTWAIT | MSG_DATAGRAM | MSG_NOSIGNAL)
547 :
548 : /* Parse control messages and extract stream or handshake metadata from msghdr. */
549 2122579 : static int quic_msghdr_parse(struct sock *sk, struct msghdr *msg, struct quic_handshake_info *hinfo,
550 : struct quic_stream_info *sinfo, bool *has_hinfo)
551 : {
552 2122579 : struct quic_handshake_info *h = NULL;
553 2122579 : struct quic_stream_info *s = NULL;
554 2122579 : struct quic_stream_table *streams;
555 2122579 : struct cmsghdr *cmsg;
556 2122579 : s64 active;
557 :
558 2122579 : if (msg->msg_flags & ~QUIC_MSG_FLAGS) /* Reject unsupported flags. */
559 : return -EINVAL;
560 :
561 2122579 : if (quic_is_closed(sk))
562 : return -EPIPE;
563 :
564 2122579 : sinfo->stream_id = -1;
565 : /* Iterate over control messages and parse recognized QUIC-level metadata. */
566 8490100 : for_each_cmsghdr(cmsg, msg) {
567 2122471 : if (!CMSG_OK(msg, cmsg))
568 : return -EINVAL;
569 :
570 2122471 : if (cmsg->cmsg_level != SOL_QUIC)
571 0 : continue;
572 :
573 2122471 : switch (cmsg->cmsg_type) {
574 3468 : case QUIC_HANDSHAKE_INFO:
575 3468 : if (cmsg->cmsg_len != CMSG_LEN(sizeof(*h)))
576 : return -EINVAL;
577 3468 : h = CMSG_DATA(cmsg);
578 3468 : hinfo->crypto_level = h->crypto_level;
579 3468 : break;
580 2119003 : case QUIC_STREAM_INFO:
581 2119003 : if (cmsg->cmsg_len != CMSG_LEN(sizeof(*s)))
582 : return -EINVAL;
583 2119003 : s = CMSG_DATA(cmsg);
584 2119003 : if (s->stream_flags & ~QUIC_MSG_STREAM_FLAGS)
585 : return -EINVAL;
586 2119003 : sinfo->stream_id = s->stream_id;
587 2119003 : sinfo->stream_flags = s->stream_flags;
588 2119003 : break;
589 : default:
590 : return -EINVAL;
591 : }
592 : }
593 :
594 2122579 : if (h) { /* If handshake metadata was provided, skip stream handling. */
595 3468 : *has_hinfo = true;
596 3468 : return 0;
597 : }
598 :
599 2119111 : if (!s) /* If no stream info was provided, inherit stream_flags from msg_flags. */
600 108 : sinfo->stream_flags |= msg->msg_flags;
601 :
602 2119111 : if (sinfo->stream_id != -1)
603 : return 0;
604 :
605 : /* No explicit stream, fallback to the active stream (the most recently opened stream). */
606 124 : streams = quic_streams(sk);
607 124 : active = streams->send.active_stream_id;
608 124 : if (active != -1) {
609 52 : sinfo->stream_id = active;
610 52 : return 0;
611 : }
612 : /* No active stream, pick the next to open based on stream direction. */
613 72 : sinfo->stream_id = streams->send.next_bidi_stream_id;
614 72 : if (sinfo->stream_flags & MSG_STREAM_UNI)
615 8 : sinfo->stream_id = streams->send.next_uni_stream_id;
616 : return 0;
617 : }
618 :
619 : /* Returns 1 if stream_id is within allowed limits or 0 otherwise. If MSG_STREAM_SNDBLOCK is
620 : * set, may send a STREAMS_BLOCKED frame.
621 : */
622 3806 : static int quic_sock_stream_available(struct sock *sk, s64 stream_id, u32 flags)
623 : {
624 3806 : struct quic_stream_table *streams = quic_streams(sk);
625 3806 : u8 type, blocked;
626 :
627 3806 : if (!quic_stream_id_send_exceeds(streams, stream_id))
628 : return 1;
629 :
630 2925 : if (!(flags & MSG_STREAM_SNDBLOCK))
631 : return 0;
632 :
633 0 : blocked = streams->send.bidi_blocked;
634 0 : type = QUIC_FRAME_STREAMS_BLOCKED_BIDI;
635 0 : if (stream_id & QUIC_STREAM_TYPE_UNI_MASK) {
636 0 : blocked = streams->send.uni_blocked;
637 0 : type = QUIC_FRAME_STREAMS_BLOCKED_UNI;
638 : }
639 :
640 0 : if (!blocked)
641 0 : quic_outq_transmit_frame(sk, type, &stream_id, 0, false);
642 : return 0;
643 : }
644 :
645 : /* Wait until the given stream ID becomes available for sending. */
646 897 : static int quic_wait_for_stream(struct sock *sk, s64 stream_id, u32 flags)
647 : {
648 897 : long timeo = sock_sndtimeo(sk, flags & MSG_STREAM_DONTWAIT);
649 897 : DEFINE_WAIT(wait);
650 897 : int err = 0;
651 :
652 2925 : for (;;) {
653 2925 : prepare_to_wait_exclusive(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
654 2925 : if (quic_is_closed(sk)) {
655 0 : err = -EPIPE;
656 0 : pr_debug("%s: sk closed\n", __func__);
657 : break;
658 : }
659 2925 : if (sk->sk_err) {
660 0 : err = -EPIPE;
661 0 : pr_debug("%s: sk_err: %d\n", __func__, sk->sk_err);
662 : break;
663 : }
664 2925 : if (signal_pending(current)) {
665 0 : err = sock_intr_errno(timeo);
666 : break;
667 : }
668 2925 : if (!timeo) {
669 : err = -EAGAIN;
670 : break;
671 : }
672 2909 : if (quic_sock_stream_available(sk, stream_id, flags))
673 : break;
674 :
675 2028 : release_sock(sk);
676 2028 : timeo = schedule_timeout(timeo);
677 4953 : lock_sock(sk);
678 : }
679 897 : finish_wait(sk_sleep(sk), &wait);
680 897 : return err;
681 : }
682 :
683 : /* Get the send stream object for the given stream ID. May wait if the stream isn't
684 : * immediately available.
685 : */
686 2127929 : static struct quic_stream *quic_sock_send_stream(struct sock *sk, struct quic_stream_info *sinfo)
687 : {
688 2127929 : struct quic_crypto *crypto = quic_crypto(sk, QUIC_CRYPTO_APP);
689 2127929 : struct quic_stream_table *streams = quic_streams(sk);
690 2127929 : struct quic_stream *stream;
691 2127929 : int err;
692 :
693 2127929 : stream = quic_stream_send_get(streams, sinfo->stream_id,
694 2127929 : sinfo->stream_flags, quic_is_serv(sk));
695 2127929 : if (!IS_ERR(stream)) {
696 2127000 : if (stream->send.state >= QUIC_STREAM_SEND_STATE_SENT)
697 : return ERR_PTR(-EINVAL); /* Can't send on a closed or finished stream. */
698 2126992 : return stream;
699 929 : } else if (PTR_ERR(stream) != -EAGAIN) {
700 : return stream;
701 : }
702 :
703 : /* App send keys are not ready yet, likely sending 0-RTT data. Do not wait for stream
704 : * availability if it's beyond the current limit; return an error immediately instead.
705 : */
706 897 : if (!crypto->send_ready)
707 : return ERR_PTR(-EINVAL);
708 :
709 897 : if (!quic_sock_stream_available(sk, sinfo->stream_id, sinfo->stream_flags)) {
710 897 : err = quic_wait_for_stream(sk, sinfo->stream_id, sinfo->stream_flags);
711 897 : if (err)
712 16 : return ERR_PTR(err);
713 : }
714 :
715 : /* Stream should now be available, retry getting the stream. */
716 881 : return quic_stream_send_get(streams, sinfo->stream_id,
717 881 : sinfo->stream_flags, quic_is_serv(sk));
718 : }
719 :
720 : /* Wait until send buffer has enough space for sending. */
721 0 : static int quic_wait_for_send(struct sock *sk, u32 flags, u32 len)
722 : {
723 0 : long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
724 0 : DEFINE_WAIT(wait);
725 0 : int err = 0;
726 :
727 0 : for (;;) {
728 0 : prepare_to_wait_exclusive(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
729 0 : if (quic_is_closed(sk)) {
730 0 : err = -EPIPE;
731 0 : pr_debug("%s: sk closed\n", __func__);
732 : break;
733 : }
734 0 : if (sk->sk_err) {
735 0 : err = -EPIPE;
736 0 : pr_debug("%s: sk_err: %d\n", __func__, sk->sk_err);
737 : break;
738 : }
739 0 : if (signal_pending(current)) {
740 0 : err = sock_intr_errno(timeo);
741 : break;
742 : }
743 0 : if (!timeo) {
744 : err = -EAGAIN;
745 : break;
746 : }
747 0 : if ((int)len <= sk_stream_wspace(sk) && sk_wmem_schedule(sk, (int)len))
748 : break;
749 :
750 0 : release_sock(sk);
751 0 : timeo = schedule_timeout(timeo);
752 0 : lock_sock(sk);
753 : }
754 0 : finish_wait(sk_sleep(sk), &wait);
755 0 : return err;
756 : }
757 :
758 : /* Check if a QUIC stream is writable. */
759 8310934 : static int quic_sock_stream_writable(struct sock *sk, struct quic_stream *stream,
760 : u32 flags, u32 len)
761 : {
762 : /* Check if flow control limits allow sending 'len' bytes. */
763 8310934 : if (quic_outq_flow_control(sk, stream, len, flags & MSG_STREAM_SNDBLOCK))
764 : return 0;
765 : /* Check socket send buffer space and memory scheduling capacity. */
766 8295637 : if (sk_stream_wspace(sk) < len || !sk_wmem_schedule(sk, len))
767 3801 : return 0;
768 : return 1;
769 : }
770 :
771 : /* Wait until a QUIC stream is writable for sending data. */
772 9380 : static int quic_wait_for_stream_send(struct sock *sk, struct quic_stream *stream, u32 flags,
773 : u32 len)
774 : {
775 9380 : long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
776 9380 : DEFINE_WAIT(wait);
777 9380 : int err = 0;
778 :
779 19098 : for (;;) {
780 19098 : prepare_to_wait_exclusive(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
781 19098 : if (quic_is_closed(sk)) {
782 0 : err = -EPIPE;
783 0 : pr_debug("%s: sk closed\n", __func__);
784 : break;
785 : }
786 19098 : if (sk->sk_err) {
787 0 : err = -EPIPE;
788 0 : pr_debug("%s: sk_err: %d\n", __func__, sk->sk_err);
789 : break;
790 : }
791 19098 : if (signal_pending(current)) {
792 0 : err = sock_intr_errno(timeo);
793 : break;
794 : }
795 19098 : if (!timeo) {
796 0 : err = -EAGAIN;
797 : /* If the stream is blocked due to flow control limits (not socket
798 : * buffer), return ENOSPC instead. This distinction helps applications
799 : * detect when they should switch to sending on other streams (e.g., to
800 : * implement fair scheduling).
801 : */
802 0 : if (quic_outq_wspace(sk, stream) < (u64)len)
803 0 : err = -ENOSPC;
804 : break;
805 : }
806 19098 : if (quic_sock_stream_writable(sk, stream, flags, len))
807 : break;
808 :
809 9718 : release_sock(sk);
810 9718 : timeo = schedule_timeout(timeo);
811 28816 : lock_sock(sk);
812 : }
813 9380 : finish_wait(sk_sleep(sk), &wait);
814 9380 : return err;
815 : }
816 :
817 2122579 : static int quic_sendmsg(struct sock *sk, struct msghdr *msg, size_t msg_len)
818 : {
819 2122579 : struct quic_outqueue *outq = quic_outq(sk);
820 2122579 : struct quic_handshake_info hinfo = {};
821 2122579 : struct quic_stream_info sinfo = {};
822 2122579 : int err = 0, bytes = 0, len = 1;
823 2122579 : bool delay, has_hinfo = false;
824 2122579 : struct quic_msginfo msginfo;
825 2122579 : struct quic_crypto *crypto;
826 2122579 : struct quic_stream *stream;
827 2122579 : u32 flags = msg->msg_flags;
828 2122579 : struct quic_frame *frame;
829 :
830 2122579 : lock_sock(sk);
831 2122579 : err = quic_msghdr_parse(sk, msg, &hinfo, &sinfo, &has_hinfo);
832 2122579 : if (err)
833 0 : goto err;
834 :
835 2122579 : delay = !!(flags & MSG_MORE); /* Determine if this is a delayed send. */
836 2122579 : if (has_hinfo) { /* Handshake Messages Send Path. */
837 : /* Initial, Handshake and App (TLS NewSessionTicket) only. */
838 3468 : if (hinfo.crypto_level >= QUIC_CRYPTO_EARLY) {
839 0 : err = -EINVAL;
840 0 : goto err;
841 : }
842 3468 : crypto = quic_crypto(sk, hinfo.crypto_level);
843 3468 : if (!crypto->send_ready) { /* Can't send if crypto keys aren't ready. */
844 0 : err = -EINVAL;
845 0 : goto err;
846 : }
847 : /* Set packet context (overhead, MSS, etc.) before fragmentation. */
848 3468 : if (quic_packet_config(sk, hinfo.crypto_level, 0)) {
849 0 : err = -ENETUNREACH;
850 0 : goto err;
851 : }
852 :
853 : /* Prepare the message info used by the frame creator. */
854 3468 : msginfo.level = hinfo.crypto_level;
855 3468 : msginfo.msg = &msg->msg_iter;
856 : /* Keep sending until all data from the message iterator is consumed. */
857 6954 : while (iov_iter_count(&msg->msg_iter) > 0) {
858 3486 : if (sk_stream_wspace(sk) < len || !sk_wmem_schedule(sk, len)) {
859 0 : if (delay) { /* Push buffered data if MSG_MORE was used. */
860 0 : outq->force_delay = 0;
861 0 : quic_outq_transmit(sk);
862 : }
863 0 : err = quic_wait_for_send(sk, flags, len);
864 0 : if (err) {
865 : /* Return error only if EPIPE or nothing was sent. */
866 0 : if (err == -EPIPE || !bytes)
867 0 : goto err;
868 0 : goto out;
869 : }
870 : }
871 3486 : frame = quic_frame_create(sk, QUIC_FRAME_CRYPTO, &msginfo);
872 3486 : if (!frame) {
873 0 : if (!bytes) { /* Return error only if nothing was sent. */
874 0 : err = -ENOMEM;
875 0 : goto err;
876 : }
877 0 : goto out;
878 : }
879 3486 : len = frame->bytes;
880 3486 : if (!sk_wmem_schedule(sk, len)) {
881 : /* Memory pressure: roll back the iterator and discard the frame. */
882 0 : iov_iter_revert(msginfo.msg, len);
883 0 : quic_frame_put(frame);
884 0 : continue; /* Go back to next frame check with len = frame->bytes. */
885 : }
886 3486 : bytes += frame->bytes;
887 3486 : outq->force_delay = delay; /* Pass the delay flag to outqueue. */
888 3486 : crypto->send_offset += frame->bytes; /* Advance crypto offset. */
889 3486 : quic_outq_ctrl_tail(sk, frame, delay); /* Queue frame for transmission. */
890 3486 : len = 1; /* Reset minimal length guess for next frame check. */
891 : }
892 3468 : goto out;
893 : }
894 :
895 2119111 : if (quic_packet_config(sk, QUIC_CRYPTO_APP, 0)) {
896 0 : err = -ENETUNREACH;
897 0 : goto err;
898 : }
899 :
900 2119111 : if (flags & MSG_DATAGRAM) { /* Datagram Messages Send Path. */
901 12 : if (!outq->max_datagram_frame_size) { /* Peer doesn't allow datagrams. */
902 0 : err = -EINVAL;
903 0 : goto err;
904 : }
905 12 : len = iov_iter_count(&msg->msg_iter);
906 12 : if (sk_stream_wspace(sk) < len || !sk_wmem_schedule(sk, len)) {
907 0 : err = quic_wait_for_send(sk, flags, len);
908 0 : if (err) {
909 0 : quic_frame_put(frame);
910 0 : goto err;
911 : }
912 : }
913 : /* Only sending Datagram frames with a length field is supported for now. */
914 12 : frame = quic_frame_create(sk, QUIC_FRAME_DATAGRAM_LEN, &msg->msg_iter);
915 12 : if (!frame) {
916 4 : err = -EINVAL;
917 4 : goto err;
918 : }
919 8 : bytes += frame->bytes;
920 8 : outq->force_delay = delay;
921 8 : quic_outq_dgram_tail(sk, frame, delay);
922 8 : goto out;
923 : }
924 :
925 : /* Stream Messages Send Path. */
926 2119099 : stream = quic_sock_send_stream(sk, &sinfo);
927 2119099 : if (IS_ERR(stream)) {
928 44 : err = PTR_ERR(stream);
929 44 : goto err;
930 : }
931 :
932 : /* Logic is similar to handshake messages send path. */
933 2119055 : msginfo.stream = stream;
934 2119055 : msginfo.msg = &msg->msg_iter;
935 2119055 : msginfo.flags = sinfo.stream_flags;
936 2119055 : flags |= sinfo.stream_flags;
937 :
938 8291836 : do {
939 8291836 : if (!quic_sock_stream_writable(sk, stream, flags, len)) {
940 9380 : if (delay) {
941 0 : outq->force_delay = 0;
942 0 : quic_outq_transmit(sk);
943 : }
944 9380 : err = quic_wait_for_stream_send(sk, stream, flags, len);
945 9380 : if (err) {
946 0 : if (err == -EPIPE || !bytes)
947 0 : goto err;
948 0 : goto out;
949 : }
950 : }
951 :
952 8291836 : len = quic_outq_stream_append(sk, &msginfo, 0); /* Probe appendable size. */
953 8291836 : if (len >= 0) {
954 1961831 : if (!sk_wmem_schedule(sk, len))
955 0 : continue; /* Memory pressure: Retry with new len. */
956 1961831 : bytes += quic_outq_stream_append(sk, &msginfo, 1); /* Appended. */
957 1961831 : len = 1; /* Reset minimal length guess for next frame check. */
958 1961831 : continue;
959 : }
960 :
961 6330005 : frame = quic_frame_create(sk, QUIC_FRAME_STREAM, &msginfo);
962 6330005 : if (!frame) {
963 0 : if (!bytes) {
964 0 : err = -ENOMEM;
965 0 : goto err;
966 : }
967 0 : goto out;
968 : }
969 6330005 : len = frame->bytes;
970 6330005 : if (!sk_wmem_schedule(sk, len)) {
971 0 : iov_iter_revert(msginfo.msg, len);
972 0 : quic_frame_put(frame);
973 0 : continue;
974 : }
975 6330005 : bytes += frame->bytes;
976 6330005 : outq->force_delay = delay;
977 6330005 : quic_outq_stream_tail(sk, frame, delay);
978 6330005 : len = 1;
979 : /* Checking iov_iter_count() after sending allows a FIN-only Stream frame. */
980 8291836 : } while (iov_iter_count(msginfo.msg) > 0);
981 2119055 : out:
982 : err = bytes; /* Return total bytes sent. */
983 2122575 : err:
984 2122579 : if (err < 0 && !has_hinfo && !(flags & MSG_DATAGRAM))
985 44 : err = sk_stream_error(sk, flags, err); /* Handle error and possibly send SIGPIPE. */
986 2122579 : release_sock(sk);
987 2122579 : return err;
988 : }
989 :
990 : /* Wait for an incoming QUIC packet. */
991 694096 : static int quic_wait_for_packet(struct sock *sk, int nonblock)
992 : {
993 694096 : struct list_head *head = &quic_inq(sk)->recv_list;
994 694096 : long timeo = sock_rcvtimeo(sk, nonblock);
995 694096 : DEFINE_WAIT(wait);
996 694096 : int err = 0;
997 :
998 695141 : for (;;) {
999 695141 : if (!list_empty(head))
1000 : break;
1001 140815 : prepare_to_wait_exclusive(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
1002 140816 : if (quic_is_closed(sk)) {
1003 375 : err = -ENOTCONN;
1004 375 : pr_debug("%s: sk closed\n", __func__);
1005 : break;
1006 : }
1007 140441 : if (sk->sk_err) {
1008 0 : err = -ENOTCONN;
1009 0 : pr_debug("%s: sk_err: %d\n", __func__, sk->sk_err);
1010 : break;
1011 : }
1012 140441 : if (signal_pending(current)) {
1013 3 : err = sock_intr_errno(timeo);
1014 : break;
1015 : }
1016 140438 : if (!timeo) {
1017 : err = -EAGAIN;
1018 : break;
1019 : }
1020 :
1021 1045 : release_sock(sk);
1022 1045 : timeo = schedule_timeout(timeo);
1023 696186 : lock_sock(sk);
1024 : }
1025 694097 : finish_wait(sk_sleep(sk), &wait);
1026 694097 : return err;
1027 : }
1028 :
1029 694096 : static int quic_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags,
1030 : int *addr_len)
1031 : {
1032 694096 : u32 copy, copied = 0, freed = 0, bytes = 0;
1033 694096 : struct quic_handshake_info hinfo = {};
1034 694096 : int nonblock = flags & MSG_DONTWAIT;
1035 694096 : struct quic_stream_info sinfo = {};
1036 694096 : struct quic_stream *stream = NULL;
1037 694096 : struct quic_frame *frame, *next;
1038 694096 : struct list_head *head;
1039 694096 : int err, fin;
1040 :
1041 694096 : lock_sock(sk);
1042 :
1043 694098 : err = quic_wait_for_packet(sk, nonblock);
1044 694097 : if (err)
1045 139772 : goto out;
1046 :
1047 554325 : head = &quic_inq(sk)->recv_list;
1048 : /* Iterate over each received frame in the list. */
1049 6370270 : list_for_each_entry_safe(frame, next, head, list) {
1050 : /* Determine how much data to copy: the minimum of the remaining data in the frame
1051 : * and the remaining user buffer space.
1052 : */
1053 6370270 : copy = min((u32)(frame->len - frame->offset), (u32)(len - copied));
1054 6370270 : if (copy) { /* Copy data from frame to user message iterator. */
1055 6370266 : copy = copy_to_iter(frame->data + frame->offset, copy, &msg->msg_iter);
1056 6370267 : if (!copy) {
1057 0 : if (!copied) { /* Return error only if nothing was coplied. */
1058 0 : err = -EFAULT;
1059 0 : goto out;
1060 : }
1061 : break;
1062 : }
1063 6370267 : copied += copy; /* Accumulate total copied bytes. */
1064 : }
1065 6370271 : fin = frame->stream_fin;
1066 6370271 : stream = frame->stream;
1067 6370271 : if (frame->event) {
1068 68 : msg->msg_flags |= MSG_NOTIFICATION; /* An Event received. */
1069 6370203 : } else if (frame->level) {
1070 : /* Attach handshake info control message if crypto level present. */
1071 2724 : hinfo.crypto_level = frame->level;
1072 2724 : put_cmsg(msg, SOL_QUIC, QUIC_HANDSHAKE_INFO, sizeof(hinfo), &hinfo);
1073 2724 : if (msg->msg_flags & MSG_CTRUNC) {
1074 0 : err = -EINVAL;
1075 0 : goto out;
1076 : }
1077 6367479 : } else if (frame->dgram) {
1078 8 : msg->msg_flags |= MSG_DATAGRAM; /* A Datagram Message received. */
1079 : }
1080 6370271 : if (flags & MSG_PEEK) /* For peek, only look at first frame, don't consume data. */
1081 : break;
1082 6370271 : if (copy != frame->len - frame->offset) {
1083 : /* Partial copy, update offset for next read and exit loop. */
1084 7680 : frame->offset += copy;
1085 7680 : break;
1086 : }
1087 6362591 : msg->msg_flags |= MSG_EOR;
1088 6362591 : bytes += frame->len; /* Track bytes fully consumed. */
1089 6362591 : if (frame->event || frame->level || frame->dgram) {
1090 : /* For these frame types, only read only one frame at a time. */
1091 2800 : list_del(&frame->list);
1092 2800 : quic_frame_put(frame);
1093 2800 : break;
1094 : }
1095 : /* A Stream Message received. */
1096 6359791 : freed += frame->len;
1097 6359791 : list_del(&frame->list);
1098 6359791 : quic_frame_put(frame);
1099 6359791 : if (fin) {
1100 : /* rfc9000#section-3.2:
1101 : *
1102 : * Once stream data has been delivered, the stream enters the "Data Read"
1103 : * state, which is a terminal state.
1104 : */
1105 17636 : stream->recv.state = QUIC_STREAM_RECV_STATE_READ;
1106 17636 : sinfo.stream_flags |= MSG_STREAM_FIN;
1107 17636 : break;
1108 : }
1109 :
1110 : /* Stop if next frame is not part of this stream or no more data to copy. */
1111 6342155 : if (list_entry_is_head(next, head, list) || copied >= len)
1112 : break;
1113 5816019 : if (next->event || next->dgram || !next->stream || next->stream != stream)
1114 : break;
1115 554326 : };
1116 :
1117 554326 : if (stream) {
1118 : /* Attach stream info control message if stream data was processed. */
1119 551526 : sinfo.stream_id = stream->id;
1120 551526 : put_cmsg(msg, SOL_QUIC, QUIC_STREAM_INFO, sizeof(sinfo), &sinfo);
1121 551526 : if (msg->msg_flags & MSG_CTRUNC)
1122 100 : msg->msg_flags |= sinfo.stream_flags;
1123 :
1124 : /* Update flow control accounting for freed bytes. */
1125 551526 : quic_inq_flow_control(sk, stream, freed);
1126 :
1127 : /* If stream read completed, purge and release resources. */
1128 551526 : if (stream->recv.state == QUIC_STREAM_RECV_STATE_READ) {
1129 17636 : quic_inq_stream_list_purge(sk, stream);
1130 17636 : quic_stream_recv_put(quic_streams(sk), stream, quic_is_serv(sk));
1131 : }
1132 : }
1133 :
1134 554326 : quic_inq_data_read(sk, bytes); /* Release receive memory accounting. */
1135 554326 : err = (int)copied;
1136 694098 : out:
1137 694098 : release_sock(sk);
1138 694098 : return err;
1139 : }
1140 :
1141 : /* Wait until a new connection request is available on the listen socket. */
1142 507 : static int quic_wait_for_accept(struct sock *sk, u32 flags)
1143 : {
1144 507 : long timeo = sock_sndtimeo(sk, flags & O_NONBLOCK);
1145 507 : struct list_head *head = quic_reqs(sk);
1146 507 : DEFINE_WAIT(wait);
1147 507 : int err = 0;
1148 :
1149 1007 : for (;;) {
1150 1007 : if (!list_empty(head))
1151 : break;
1152 587 : prepare_to_wait_exclusive(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
1153 587 : if (!quic_is_listen(sk)) {
1154 0 : err = -EINVAL;
1155 0 : pr_debug("%s: sk not listening\n", __func__);
1156 : break;
1157 : }
1158 587 : if (sk->sk_err) {
1159 0 : err = -EINVAL;
1160 0 : pr_debug("%s: sk_err: %d\n", __func__, sk->sk_err);
1161 : break;
1162 : }
1163 587 : if (signal_pending(current)) {
1164 86 : err = sock_intr_errno(timeo);
1165 : break;
1166 : }
1167 501 : if (!timeo) {
1168 : err = -EAGAIN;
1169 : break;
1170 : }
1171 :
1172 501 : release_sock(sk);
1173 501 : timeo = schedule_timeout(timeo);
1174 1507 : lock_sock(sk);
1175 : }
1176 506 : finish_wait(sk_sleep(sk), &wait);
1177 506 : return err;
1178 : }
1179 :
1180 : /* Apply QUIC configuration settings to a socket. */
1181 990 : static int quic_sock_apply_config(struct sock *sk, struct quic_config *c)
1182 : {
1183 990 : struct quic_config *config = quic_config(sk);
1184 990 : struct quic_packet *packet = quic_packet(sk);
1185 990 : struct quic_cong *cong = quic_cong(sk);
1186 :
1187 990 : if (c->validate_peer_address)
1188 8 : config->validate_peer_address = c->validate_peer_address;
1189 990 : if (c->receive_session_ticket)
1190 2 : config->receive_session_ticket = c->receive_session_ticket;
1191 990 : if (c->certificate_request)
1192 0 : config->certificate_request = c->certificate_request;
1193 990 : if (c->initial_smoothed_rtt) {
1194 916 : if (c->initial_smoothed_rtt < QUIC_RTT_MIN ||
1195 : c->initial_smoothed_rtt > QUIC_RTT_MAX)
1196 : return -EINVAL;
1197 916 : config->initial_smoothed_rtt = c->initial_smoothed_rtt;
1198 916 : quic_cong_set_srtt(cong, config->initial_smoothed_rtt);
1199 : }
1200 990 : if (c->plpmtud_probe_interval) {
1201 8 : if (c->plpmtud_probe_interval < QUIC_MIN_PROBE_TIMEOUT)
1202 : return -EINVAL;
1203 8 : config->plpmtud_probe_interval = c->plpmtud_probe_interval;
1204 : }
1205 990 : if (c->payload_cipher_type) {
1206 0 : if (c->payload_cipher_type != TLS_CIPHER_AES_GCM_128 &&
1207 : c->payload_cipher_type != TLS_CIPHER_AES_GCM_256 &&
1208 0 : c->payload_cipher_type != TLS_CIPHER_AES_CCM_128 &&
1209 : c->payload_cipher_type != TLS_CIPHER_CHACHA20_POLY1305)
1210 : return -EINVAL;
1211 0 : config->payload_cipher_type = c->payload_cipher_type;
1212 : }
1213 990 : if (c->version) {
1214 6 : config->version = c->version;
1215 6 : packet->version = c->version;
1216 : }
1217 990 : if (c->congestion_control_algo) {
1218 8 : config->congestion_control_algo = c->congestion_control_algo;
1219 8 : quic_cong_set_algo(cong, config->congestion_control_algo);
1220 : }
1221 990 : if (c->stream_data_nodelay)
1222 0 : config->stream_data_nodelay = c->stream_data_nodelay;
1223 :
1224 : return 0;
1225 : }
1226 :
1227 : /* Initialize an accept QUIC socket from a listen socket and a connection request. */
1228 420 : static int quic_copy_sock(struct sock *nsk, struct sock *sk, struct quic_request_sock *req)
1229 : {
1230 420 : struct quic_pnspace *space = quic_pnspace(sk, QUIC_CRYPTO_INITIAL);
1231 420 : struct quic_packet *packet = quic_packet(sk);
1232 420 : struct quic_inqueue *inq = quic_inq(sk);
1233 420 : struct quic_transport_param param = {};
1234 420 : struct sk_buff *skb, *tmp;
1235 :
1236 : /* Duplicate ALPN from listen to accept socket for handshake. */
1237 420 : if (quic_data_dup(quic_alpn(nsk), quic_alpn(sk)->data, quic_alpn(sk)->len))
1238 : return -ENOMEM;
1239 :
1240 : /* Copy socket metadata. */
1241 420 : nsk->sk_type = sk->sk_type;
1242 420 : nsk->sk_flags = sk->sk_flags;
1243 420 : nsk->sk_protocol = IPPROTO_QUIC;
1244 420 : nsk->sk_backlog_rcv = sk->sk_prot->backlog_rcv;
1245 :
1246 420 : nsk->sk_sndbuf = sk->sk_sndbuf;
1247 420 : nsk->sk_rcvbuf = sk->sk_rcvbuf;
1248 420 : nsk->sk_rcvtimeo = sk->sk_rcvtimeo;
1249 420 : nsk->sk_sndtimeo = sk->sk_sndtimeo;
1250 :
1251 420 : inet_sk(nsk)->pmtudisc = inet_sk(sk)->pmtudisc;
1252 :
1253 : /* Move matching packets from listen socket's backlog to accept socket. */
1254 906 : skb_queue_walk_safe(&inq->backlog_list, skb, tmp) {
1255 486 : quic_get_msg_addrs(&packet->saddr, &packet->daddr, skb);
1256 486 : quic_packet_get_dcid(&packet->dcid, skb);
1257 972 : if (!memcmp(&req->saddr, &packet->saddr, sizeof(req->saddr)) &&
1258 1458 : !memcmp(&req->daddr, &packet->daddr, sizeof(req->daddr)) &&
1259 486 : !quic_conn_id_cmp(&req->dcid, &packet->dcid)) {
1260 486 : __skb_unlink(skb, &inq->backlog_list);
1261 486 : quic_inq_backlog_tail(nsk, skb);
1262 : }
1263 : }
1264 :
1265 : /* Record the creation time of this accept socket in microseconds. Used by
1266 : * quic_accept_sock_exists() to determine if a packet from sk_backlog of
1267 : * listen socket predates this socket.
1268 : */
1269 420 : space->time = jiffies_to_usecs(jiffies);
1270 :
1271 420 : if (sk->sk_family == AF_INET6) /* Set IPv6 specific state if applicable. */
1272 392 : inet_sk(nsk)->pinet6 = &((struct quic6_sock *)nsk)->inet6;
1273 :
1274 420 : quic_inq(nsk)->events = inq->events;
1275 420 : quic_paths(nsk)->serv = 1; /* Mark this as a server. */
1276 :
1277 : /* Copy the QUIC settings and transport parameters to accept socket. */
1278 420 : quic_sock_apply_config(nsk, quic_config(sk));
1279 420 : quic_sock_fetch_transport_param(sk, ¶m);
1280 420 : quic_sock_apply_transport_param(nsk, ¶m);
1281 :
1282 420 : return 0;
1283 : }
1284 :
1285 : /* Finalize setup for an accept QUIC socket. */
1286 420 : static int quic_accept_sock_setup(struct sock *sk, struct quic_request_sock *req)
1287 : {
1288 420 : struct quic_crypto *crypto = quic_crypto(sk, QUIC_CRYPTO_INITIAL);
1289 420 : struct quic_path_group *paths = quic_paths(sk);
1290 420 : struct quic_packet *packet = quic_packet(sk);
1291 420 : struct quic_inqueue *inq = quic_inq(sk);
1292 420 : struct quic_conn_id conn_id;
1293 420 : struct sk_buff_head tmpq;
1294 420 : struct sk_buff *skb;
1295 420 : int err;
1296 :
1297 420 : lock_sock(sk);
1298 : /* Set destination address and resolve route (may also auto-set source address). */
1299 420 : quic_path_set_daddr(paths, 0, &req->daddr);
1300 420 : err = quic_packet_route(sk);
1301 420 : if (err < 0)
1302 0 : goto out;
1303 420 : quic_set_sk_addr(sk, &req->daddr, false);
1304 :
1305 : /* Generate and add destination and source connection IDs. */
1306 420 : quic_conn_id_generate(&conn_id);
1307 420 : err = quic_conn_id_add(quic_source(sk), &conn_id, 0, sk);
1308 420 : if (err)
1309 0 : goto out;
1310 420 : err = quic_conn_id_add(quic_dest(sk), &req->scid, 0, NULL);
1311 420 : if (err)
1312 0 : goto out;
1313 :
1314 : /* Install initial encryption keys for handshake. */
1315 420 : err = quic_crypto_set_cipher(crypto, TLS_CIPHER_AES_GCM_128, CRYPTO_ALG_ASYNC);
1316 420 : if (err)
1317 0 : goto out;
1318 420 : err = quic_crypto_initial_keys_install(crypto, &req->dcid, req->version, 1);
1319 420 : if (err)
1320 0 : goto out;
1321 : /* Record the QUIC version offered by the peer. May later change if Compatible Version
1322 : * Negotiation is triggered.
1323 : */
1324 420 : packet->version = req->version;
1325 :
1326 : /* Save original DCID and retry DCID for building transport parameters, and identifying
1327 : * the connection in quic_sock_lookup().
1328 : */
1329 420 : paths->orig_dcid = req->orig_dcid;
1330 420 : if (req->retry) {
1331 4 : paths->retry = 1;
1332 4 : paths->retry_dcid = req->dcid;
1333 : }
1334 :
1335 : /* Add socket to hash table, change state to ESTABLISHING, and start idle timer. */
1336 420 : err = sk->sk_prot->hash(sk);
1337 420 : if (err)
1338 0 : goto out;
1339 420 : quic_set_state(sk, QUIC_SS_ESTABLISHING);
1340 420 : quic_timer_start(sk, QUIC_TIMER_IDLE, inq->timeout);
1341 :
1342 : /* Process all packets in backlog list of this socket. */
1343 420 : __skb_queue_head_init(&tmpq);
1344 420 : skb_queue_splice_init(&inq->backlog_list, &tmpq);
1345 420 : skb = __skb_dequeue(&tmpq);
1346 906 : while (skb) {
1347 486 : quic_packet_process(sk, skb);
1348 486 : skb = __skb_dequeue(&tmpq);
1349 : }
1350 :
1351 420 : out:
1352 420 : release_sock(sk);
1353 420 : return err;
1354 : }
1355 :
1356 507 : static struct sock *quic_accept(struct sock *sk, struct proto_accept_arg *arg)
1357 : {
1358 507 : struct quic_request_sock *req = NULL;
1359 507 : struct sock *nsk = NULL;
1360 507 : int err = -EINVAL;
1361 :
1362 507 : lock_sock(sk);
1363 :
1364 507 : if (!quic_is_listen(sk))
1365 0 : goto out;
1366 :
1367 507 : err = quic_wait_for_accept(sk, arg->flags);
1368 506 : if (err)
1369 86 : goto out;
1370 420 : req = quic_request_sock_dequeue(sk);
1371 :
1372 420 : nsk = sk_alloc(sock_net(sk), sk->sk_family, GFP_KERNEL, sk->sk_prot, arg->kern);
1373 420 : if (!nsk) {
1374 0 : err = -ENOMEM;
1375 0 : goto out;
1376 : }
1377 420 : sock_init_data(NULL, nsk);
1378 :
1379 420 : err = nsk->sk_prot->init(nsk);
1380 420 : if (err)
1381 0 : goto free;
1382 :
1383 420 : err = quic_copy_sock(nsk, sk, req);
1384 420 : if (err)
1385 0 : goto free;
1386 420 : err = nsk->sk_prot->bind(nsk, &req->saddr.sa, sizeof(req->saddr));
1387 420 : if (err)
1388 0 : goto free;
1389 :
1390 420 : err = quic_accept_sock_setup(nsk, req);
1391 420 : if (err)
1392 0 : goto free;
1393 :
1394 420 : out:
1395 506 : release_sock(sk);
1396 506 : arg->err = err;
1397 506 : kfree(req);
1398 506 : return nsk;
1399 0 : free:
1400 0 : nsk->sk_prot->close(nsk, 0);
1401 0 : nsk = NULL;
1402 0 : goto out;
1403 : }
1404 :
1405 1118 : static void quic_close(struct sock *sk, long timeout)
1406 : {
1407 1118 : lock_sock(sk);
1408 :
1409 1118 : quic_outq_transmit_app_close(sk);
1410 1118 : quic_set_state(sk, QUIC_SS_CLOSED);
1411 :
1412 1118 : release_sock(sk);
1413 :
1414 1118 : sk_common_release(sk);
1415 1118 : }
1416 :
1417 52 : static int quic_sock_set_event(struct sock *sk, struct quic_event_option *event, u32 len)
1418 : {
1419 52 : struct quic_inqueue *inq = quic_inq(sk);
1420 :
1421 52 : if (len != sizeof(*event))
1422 : return -EINVAL;
1423 52 : if (!event->type || event->type > QUIC_EVENT_MAX)
1424 : return -EINVAL;
1425 :
1426 52 : if (event->on) { /* Enable the specified event by setting the corresponding bit. */
1427 28 : inq->events |= BIT(event->type);
1428 28 : return 0;
1429 : }
1430 24 : inq->events &= ~BIT(event->type); /* Disable the specified event by clearing the bit. */
1431 24 : return 0;
1432 : }
1433 :
1434 24 : static int quic_sock_stream_reset(struct sock *sk, struct quic_errinfo *info, u32 len)
1435 : {
1436 24 : struct quic_stream_table *streams = quic_streams(sk);
1437 24 : struct quic_stream *stream;
1438 24 : struct quic_frame *frame;
1439 :
1440 24 : if (len != sizeof(*info) || !quic_is_established(sk))
1441 : return -EINVAL;
1442 :
1443 24 : stream = quic_stream_send_get(streams, info->stream_id, 0, quic_is_serv(sk));
1444 24 : if (IS_ERR(stream))
1445 8 : return PTR_ERR(stream);
1446 :
1447 : /* rfc9000#section-3.1:
1448 : *
1449 : * From any state that is one of "Ready", "Send", or "Data Sent", an application can
1450 : * signal that it wishes to abandon transmission of stream data. The endpoint sends a
1451 : * RESET_STREAM frame, which causes the stream to enter the "Reset Sent" state.
1452 : */
1453 16 : if (stream->send.state >= QUIC_STREAM_SEND_STATE_RECVD)
1454 : return -EINVAL;
1455 :
1456 16 : frame = quic_frame_create(sk, QUIC_FRAME_RESET_STREAM, info);
1457 16 : if (!frame)
1458 : return -ENOMEM;
1459 :
1460 16 : stream->send.state = QUIC_STREAM_SEND_STATE_RESET_SENT;
1461 16 : quic_outq_stream_list_purge(sk, stream);
1462 16 : quic_outq_ctrl_tail(sk, frame, false);
1463 16 : return 0;
1464 : }
1465 :
1466 8 : static int quic_sock_stream_stop_sending(struct sock *sk, struct quic_errinfo *info, u32 len)
1467 : {
1468 8 : struct quic_stream_table *streams = quic_streams(sk);
1469 8 : struct quic_stream *stream;
1470 :
1471 8 : if (len != sizeof(*info) || !quic_is_established(sk))
1472 : return -EINVAL;
1473 :
1474 8 : stream = quic_stream_recv_get(streams, info->stream_id, quic_is_serv(sk));
1475 8 : if (IS_ERR(stream))
1476 0 : return PTR_ERR(stream);
1477 :
1478 : /* rfc9000#section-3.3:
1479 : *
1480 : * A receiver MAY send a STOP_SENDING frame in any state where it has not received a
1481 : * RESET_STREAM frame -- that is, states other than "Reset Recvd" or "Reset Read".
1482 : * However, there is little value in sending a STOP_SENDING frame in the "Data Recvd"
1483 : * state, as all stream data has been received.
1484 : */
1485 8 : if (stream->recv.state >= QUIC_STREAM_RECV_STATE_RECVD)
1486 : return -EINVAL;
1487 :
1488 8 : if (stream->send.stop_sent) /* Defer sending; a STOP_SENDING frame is already in flight. */
1489 : return -EAGAIN;
1490 :
1491 8 : return quic_outq_transmit_frame(sk, QUIC_FRAME_STOP_SENDING, info, 0, false);
1492 : }
1493 :
1494 72 : static int quic_sock_set_connection_id(struct sock *sk,
1495 : struct quic_connection_id_info *info, u32 len)
1496 : {
1497 72 : struct quic_conn_id_set *id_set = quic_source(sk);
1498 72 : struct quic_conn_id *active, *old;
1499 72 : u64 number, first, last;
1500 :
1501 72 : if (len < sizeof(*info) || !quic_is_established(sk))
1502 : return -EINVAL;
1503 :
1504 72 : if (info->dest) {
1505 36 : id_set = quic_dest(sk);
1506 : /* The alternative connection ID is reserved for the migration path. Until the
1507 : * migration completes and this path becomes active, no modifications should be
1508 : * made to the destination connection ID set until then.
1509 : */
1510 36 : if (id_set->alt)
1511 : return -EAGAIN;
1512 : }
1513 72 : old = quic_conn_id_active(id_set);
1514 72 : if (info->active) { /* Change active connection ID. */
1515 : /* Ensure the new active ID is greater than the current one. All lower-numbered
1516 : * IDs are implicitly treated as used.
1517 : */
1518 36 : if (info->active <= quic_conn_id_number(old))
1519 : return -EINVAL;
1520 28 : active = quic_conn_id_find(id_set, info->active);
1521 28 : if (!active)
1522 : return -EINVAL;
1523 20 : quic_conn_id_set_active(id_set, active);
1524 : }
1525 :
1526 56 : if (!info->prior_to)
1527 : return 0;
1528 :
1529 : /* Retire connection IDs up to (but not including) 'prior_to'. */
1530 56 : number = info->prior_to;
1531 56 : last = quic_conn_id_last_number(id_set);
1532 56 : first = quic_conn_id_first_number(id_set);
1533 56 : if (number > last || number <= first) {
1534 : /* Invalid retirement range: revert any active ID change. */
1535 16 : quic_conn_id_set_active(id_set, old);
1536 16 : return -EINVAL;
1537 : }
1538 :
1539 40 : if (!info->dest) { /* Retire source connection IDs by sending NEW_CONNECTION_ID frames. */
1540 20 : if (quic_outq_transmit_new_conn_id(sk, number, 0, false)) {
1541 0 : quic_conn_id_set_active(id_set, old);
1542 0 : return -ENOMEM;
1543 : }
1544 : return 0;
1545 : }
1546 :
1547 : /* Retire destination connection IDs by sending RETIRE_CONNECTION_ID frames. */
1548 20 : if (quic_outq_transmit_retire_conn_id(sk, number, 0, false)) {
1549 0 : quic_conn_id_set_active(id_set, old);
1550 0 : return -ENOMEM;
1551 : }
1552 :
1553 : return 0;
1554 : }
1555 :
1556 16 : static int quic_sock_set_connection_close(struct sock *sk, struct quic_connection_close *close,
1557 : u32 len)
1558 : {
1559 16 : struct quic_outqueue *outq = quic_outq(sk);
1560 16 : u8 *data;
1561 :
1562 16 : if (len < sizeof(*close))
1563 : return -EINVAL;
1564 :
1565 : /* Remaining length is the length of the phrase (if any). */
1566 12 : len -= sizeof(*close);
1567 12 : if (len > QUIC_CLOSE_PHRASE_MAX_LEN + 1)
1568 : return -EINVAL;
1569 :
1570 12 : if (len) {
1571 12 : if (close->phrase[len - 1]) /* Phrase must be ended with '\0'. */
1572 : return -EINVAL;
1573 8 : data = kmemdup(close->phrase, len, GFP_KERNEL);
1574 8 : if (!data)
1575 : return -ENOMEM;
1576 8 : kfree(outq->close_phrase);
1577 8 : outq->close_phrase = data;
1578 : }
1579 :
1580 8 : outq->close_errcode = close->errcode;
1581 8 : return 0;
1582 : }
1583 :
1584 43 : static int quic_sock_connection_migrate(struct sock *sk, struct sockaddr *addr, int addr_len)
1585 : {
1586 43 : struct quic_path_group *paths = quic_paths(sk);
1587 43 : union quic_addr a;
1588 43 : int err;
1589 :
1590 43 : if (quic_get_user_addr(sk, &a, addr, addr_len))
1591 : return -EINVAL;
1592 : /* Reject if connection is closed or address matches the current path's source. */
1593 43 : if (quic_is_closed(sk) || quic_cmp_sk_addr(sk, quic_path_saddr(paths, 0), &a))
1594 0 : return -EINVAL;
1595 :
1596 43 : if (!quic_is_established(sk)) {
1597 : /* Allows setting a preferred address before the handshake completes.
1598 : * The address may use a different address family (e.g., IPv4 vs IPv6).
1599 : */
1600 27 : if (!quic_is_serv(sk) || paths->disable_saddr_alt)
1601 : return -EINVAL;
1602 27 : paths->pref_addr = 1;
1603 27 : quic_path_set_saddr(paths, 1, &a);
1604 27 : return 0;
1605 : }
1606 :
1607 : /* Migration requires matching address family and a valid port. */
1608 16 : if (a.sa.sa_family != quic_path_saddr(paths, 0)->sa.sa_family || !a.v4.sin_port)
1609 : return -EINVAL;
1610 : /* Reject if a migration is in progress or a preferred address is already active. */
1611 16 : if (!quic_path_alt_state(paths, QUIC_PATH_ALT_NONE) || paths->pref_addr)
1612 : return -EAGAIN;
1613 :
1614 : /* Setup path 1 with the new source address and existing destination address. */
1615 16 : quic_path_set_saddr(paths, 1, &a);
1616 16 : quic_path_set_daddr(paths, 1, quic_path_daddr(paths, 0));
1617 :
1618 : /* Configure routing and bind to the new source address. */
1619 16 : if (quic_packet_config(sk, 0, 1))
1620 : return -EINVAL;
1621 16 : if (quic_path_bind(sk, paths, 1))
1622 : return -EINVAL;
1623 16 : err = quic_outq_probe_path_alt(sk, false); /* Start connection migration using new path. */
1624 16 : if (err)
1625 0 : quic_path_free(sk, paths, 1); /* Cleanup path 1 on failure. */
1626 : return err;
1627 : }
1628 :
1629 : /* Validate and copy QUIC transport parameters. */
1630 639 : static int quic_param_check_and_copy(struct quic_transport_param *p,
1631 : struct quic_transport_param *param)
1632 : {
1633 639 : if (p->max_udp_payload_size) {
1634 10 : if (p->max_udp_payload_size < QUIC_MIN_UDP_PAYLOAD ||
1635 : p->max_udp_payload_size > QUIC_MAX_UDP_PAYLOAD)
1636 : return -EINVAL;
1637 10 : param->max_udp_payload_size = p->max_udp_payload_size;
1638 : }
1639 639 : if (p->ack_delay_exponent) {
1640 10 : if (p->ack_delay_exponent > QUIC_MAX_ACK_DELAY_EXPONENT)
1641 : return -EINVAL;
1642 10 : param->ack_delay_exponent = p->ack_delay_exponent;
1643 : }
1644 639 : if (p->max_ack_delay) {
1645 570 : if (p->max_ack_delay >= QUIC_MAX_ACK_DELAY)
1646 : return -EINVAL;
1647 570 : param->max_ack_delay = p->max_ack_delay;
1648 : }
1649 639 : if (p->active_connection_id_limit) {
1650 10 : if (p->active_connection_id_limit < QUIC_CONN_ID_LEAST ||
1651 : p->active_connection_id_limit > QUIC_CONN_ID_LIMIT)
1652 : return -EINVAL;
1653 10 : param->active_connection_id_limit = p->active_connection_id_limit;
1654 : }
1655 639 : if (p->max_idle_timeout) {
1656 594 : if (p->max_idle_timeout < QUIC_MIN_IDLE_TIMEOUT)
1657 : return -EINVAL;
1658 594 : param->max_idle_timeout = p->max_idle_timeout;
1659 : }
1660 639 : if (p->max_datagram_frame_size) {
1661 8 : if (p->max_datagram_frame_size < QUIC_PATH_MIN_PMTU)
1662 : return -EINVAL;
1663 8 : param->max_datagram_frame_size = p->max_datagram_frame_size;
1664 : }
1665 639 : if (p->max_data) {
1666 10 : if (p->max_data < QUIC_PATH_MIN_PMTU ||
1667 10 : (!p->remote && p->max_data > (S32_MAX / 2)))
1668 : return -EINVAL;
1669 10 : param->max_data = p->max_data;
1670 : }
1671 639 : if (p->max_stream_data_bidi_local) {
1672 9 : if (p->max_stream_data_bidi_local < QUIC_PATH_MIN_PMTU ||
1673 9 : (!p->remote && p->max_stream_data_bidi_local > (S32_MAX / 4)))
1674 : return -EINVAL;
1675 9 : param->max_stream_data_bidi_local = p->max_stream_data_bidi_local;
1676 : }
1677 639 : if (p->max_stream_data_bidi_remote) {
1678 10 : if (p->max_stream_data_bidi_remote < QUIC_PATH_MIN_PMTU ||
1679 10 : (!p->remote && p->max_stream_data_bidi_remote > (S32_MAX / 4)))
1680 : return -EINVAL;
1681 10 : param->max_stream_data_bidi_remote = p->max_stream_data_bidi_remote;
1682 : }
1683 639 : if (p->max_stream_data_uni) {
1684 10 : if (p->max_stream_data_uni < QUIC_PATH_MIN_PMTU ||
1685 10 : (!p->remote && p->max_stream_data_uni > (S32_MAX / 4)))
1686 : return -EINVAL;
1687 10 : param->max_stream_data_uni = p->max_stream_data_uni;
1688 : }
1689 639 : if (p->max_streams_bidi) {
1690 570 : if (p->max_streams_bidi > QUIC_MAX_STREAMS) {
1691 0 : if (!p->remote)
1692 : return -EINVAL;
1693 0 : p->max_streams_bidi = QUIC_MAX_STREAMS;
1694 : }
1695 570 : param->max_streams_bidi = p->max_streams_bidi;
1696 : }
1697 639 : if (p->max_streams_uni) {
1698 9 : if (p->max_streams_uni > QUIC_MAX_STREAMS) {
1699 0 : if (!p->remote)
1700 : return -EINVAL;
1701 0 : p->max_streams_uni = QUIC_MAX_STREAMS;
1702 : }
1703 9 : param->max_streams_uni = p->max_streams_uni;
1704 : }
1705 639 : if (p->disable_active_migration)
1706 0 : param->disable_active_migration = p->disable_active_migration;
1707 639 : if (p->disable_1rtt_encryption)
1708 4 : param->disable_1rtt_encryption = p->disable_1rtt_encryption;
1709 639 : if (p->disable_compatible_version)
1710 480 : param->disable_compatible_version = p->disable_compatible_version;
1711 639 : if (p->grease_quic_bit)
1712 613 : param->grease_quic_bit = p->grease_quic_bit;
1713 639 : if (p->stateless_reset)
1714 14 : param->stateless_reset = p->stateless_reset;
1715 :
1716 : return 0;
1717 : }
1718 :
1719 639 : static int quic_sock_set_transport_param(struct sock *sk, struct quic_transport_param *p, u32 len)
1720 : {
1721 639 : struct quic_transport_param param = {};
1722 :
1723 639 : if (len < sizeof(param) || quic_is_established(sk))
1724 : return -EINVAL;
1725 :
1726 : /* Manually setting remote transport parameters is required only to enable 0-RTT data
1727 : * transmission during handshake initiation.
1728 : */
1729 639 : if (p->remote && !quic_is_establishing(sk))
1730 : return -EINVAL;
1731 :
1732 639 : param.remote = p->remote;
1733 639 : quic_sock_fetch_transport_param(sk, ¶m);
1734 :
1735 639 : if (quic_param_check_and_copy(p, ¶m))
1736 : return -EINVAL;
1737 :
1738 639 : quic_sock_apply_transport_param(sk, ¶m);
1739 639 : return 0;
1740 : }
1741 :
1742 570 : static int quic_sock_set_config(struct sock *sk, struct quic_config *c, u32 len)
1743 : {
1744 570 : if (len < sizeof(*c) || quic_is_established(sk))
1745 : return -EINVAL;
1746 :
1747 570 : return quic_sock_apply_config(sk, c);
1748 : }
1749 :
1750 : #define QUIC_ALPN_MAX_LEN 128
1751 :
1752 126 : static int quic_sock_set_alpn(struct sock *sk, u8 *data, u32 len)
1753 : {
1754 126 : struct quic_data *alpns = quic_alpn(sk);
1755 126 : u8 *p;
1756 :
1757 126 : if (!len || len > QUIC_ALPN_MAX_LEN || quic_is_listen(sk))
1758 : return -EINVAL;
1759 :
1760 126 : p = kzalloc(len + 1, GFP_KERNEL);
1761 126 : if (!p)
1762 : return -ENOMEM;
1763 :
1764 126 : kfree(alpns->data);
1765 126 : alpns->data = p;
1766 126 : alpns->len = len + 1;
1767 :
1768 126 : quic_data_from_string(alpns, data, len);
1769 126 : return 0;
1770 : }
1771 :
1772 12 : static int quic_sock_set_token(struct sock *sk, void *data, u32 len)
1773 : {
1774 12 : if (quic_is_serv(sk)) {
1775 : /* For servers, send a regular token to client via NEW_TOKEN frames after
1776 : * handshake.
1777 : */
1778 8 : if (!quic_is_established(sk))
1779 : return -EINVAL;
1780 : /* Defer sending; a NEW_TOKEN frame is already in flight. */
1781 8 : if (quic_outq(sk)->token_pending)
1782 : return -EAGAIN;
1783 8 : if (quic_outq_transmit_frame(sk, QUIC_FRAME_NEW_TOKEN, NULL, 0, false))
1784 : return -ENOMEM;
1785 8 : return 0;
1786 : }
1787 :
1788 : /* For clients, use the regular token next time before handshake. */
1789 4 : if (!len || len > QUIC_TOKEN_MAX_LEN)
1790 : return -EINVAL;
1791 :
1792 0 : return quic_data_dup(quic_token(sk), data, len);
1793 : }
1794 :
1795 4 : static int quic_sock_set_session_ticket(struct sock *sk, u8 *data, u32 len)
1796 : {
1797 4 : if (len < QUIC_TICKET_MIN_LEN || len > QUIC_TICKET_MAX_LEN)
1798 : return -EINVAL;
1799 :
1800 4 : return quic_data_dup(quic_ticket(sk), data, len);
1801 : }
1802 :
1803 : #define QUIC_TP_EXT_MAX_LEN 256
1804 :
1805 994 : static int quic_sock_set_transport_params_ext(struct sock *sk, u8 *p, u32 len)
1806 : {
1807 994 : struct quic_transport_param param = {};
1808 994 : u32 errcode;
1809 :
1810 994 : if (!quic_is_establishing(sk) || len > QUIC_TP_EXT_MAX_LEN)
1811 : return -EINVAL;
1812 :
1813 994 : param.remote = 1;
1814 994 : if (quic_frame_parse_transport_params_ext(sk, ¶m, p, len)) {
1815 0 : errcode = QUIC_TRANSPORT_ERROR_TRANSPORT_PARAM;
1816 0 : quic_outq_transmit_close(sk, 0, errcode, QUIC_CRYPTO_INITIAL);
1817 0 : return -EINVAL;
1818 : }
1819 :
1820 994 : quic_sock_apply_transport_param(sk, ¶m);
1821 994 : return 0;
1822 : }
1823 :
1824 4059 : static int quic_sock_set_crypto_secret(struct sock *sk, struct quic_crypto_secret *secret, u32 len)
1825 : {
1826 4059 : struct quic_path_group *paths = quic_paths(sk);
1827 4059 : struct quic_packet *packet = quic_packet(sk);
1828 4059 : struct quic_outqueue *outq = quic_outq(sk);
1829 4059 : struct quic_inqueue *inq = quic_inq(sk);
1830 4059 : struct quic_config *c = quic_config(sk);
1831 4059 : struct quic_crypto *crypto;
1832 4059 : struct sk_buff_head tmpq;
1833 4059 : struct sk_buff *skb;
1834 4059 : union quic_addr *a;
1835 4059 : int err;
1836 :
1837 4059 : if (!quic_is_establishing(sk) || len != sizeof(*secret))
1838 : return -EINVAL;
1839 :
1840 : /* Accept only supported levels: Handshake, 0-RTT (Early), or 1-RTT (App). The initial
1841 : * secret was already derived in-kernel using the original destination connection ID.
1842 : */
1843 4059 : if (secret->level != QUIC_CRYPTO_APP &&
1844 1988 : secret->level != QUIC_CRYPTO_EARLY &&
1845 : secret->level != QUIC_CRYPTO_HANDSHAKE)
1846 : return -EINVAL;
1847 :
1848 : /* Install keys into the crypto context. */
1849 4059 : crypto = quic_crypto(sk, secret->level);
1850 4059 : err = quic_crypto_set_secret(crypto, secret, packet->version, 0);
1851 4059 : if (err)
1852 : return err;
1853 :
1854 4059 : if (secret->level != QUIC_CRYPTO_APP) {
1855 2072 : if (secret->send) { /* 0-RTT or Handshake send key is ready. */
1856 : /* If 0-RTT send key is ready, set data_level to EARLY. This allows
1857 : * quic_outq_transmit_stream() to emit stream frames in 0-RTT packets.
1858 : */
1859 1008 : if (secret->level == QUIC_CRYPTO_EARLY)
1860 14 : outq->data_level = QUIC_CRYPTO_EARLY;
1861 1008 : return 0;
1862 : }
1863 : /* 0-RTT or Handshake receive key is ready; decrypt and process all buffered
1864 : * 0-RTT or Handshake packets.
1865 : */
1866 1064 : __skb_queue_head_init(&tmpq);
1867 1064 : skb_queue_splice_init(&inq->backlog_list, &tmpq);
1868 1064 : skb = __skb_dequeue(&tmpq);
1869 1748 : while (skb) {
1870 684 : quic_packet_process(sk, skb);
1871 684 : skb = __skb_dequeue(&tmpq);
1872 : }
1873 : return 0;
1874 : }
1875 :
1876 1987 : if (secret->send) {
1877 : /* App send key is ready, set data_level to APP. This allows
1878 : * quic_outq_transmit_stream() to emit stream frames in 1-RTT packets.
1879 : */
1880 994 : outq->data_level = QUIC_CRYPTO_APP;
1881 994 : if (!crypto->recv_ready)
1882 : return 0;
1883 0 : goto done; /* Both send and receive keys are ready; handshake complete. */
1884 : }
1885 :
1886 : /* Free previously stored TLS session ticket and QUIC token data, allowing reception of
1887 : * fresh NewSessionTicket message and regular token in NEW_TOKEN frames from the peer
1888 : * during handshake completion.
1889 : */
1890 993 : quic_data_free(quic_ticket(sk));
1891 993 : quic_data_free(quic_token(sk));
1892 993 : if (!list_empty(&inq->early_list)) {
1893 : /* If any 0-RTT data was buffered (early_list), move it to the main receive
1894 : * list (recv_list) so it becomes available to the application.
1895 : */
1896 68 : list_splice_init(&inq->early_list, &inq->recv_list);
1897 68 : sk->sk_data_ready(sk);
1898 : }
1899 : /* App receive key is ready; decrypt and process all buffered App/1-RTT packets. */
1900 993 : __skb_queue_head_init(&tmpq);
1901 993 : skb_queue_splice_init(&inq->backlog_list, &tmpq);
1902 993 : skb = __skb_dequeue(&tmpq);
1903 1467 : while (skb) {
1904 474 : quic_packet_process(sk, skb);
1905 474 : skb = __skb_dequeue(&tmpq);
1906 : }
1907 :
1908 993 : if (!crypto->send_ready)
1909 : return 0;
1910 993 : done:
1911 : /* Both send and receive keys are ready; handshake complete. */
1912 993 : if (!quic_is_serv(sk)) {
1913 574 : if (!paths->pref_addr)
1914 546 : goto out;
1915 : /* The peer offered a preferred address (stored in path 1). Reset the flag to
1916 : * avoid reprocessing, and Perform routing on new path and set the local address
1917 : * for new path.
1918 : */
1919 28 : if (quic_packet_config(sk, 0, 1)) {
1920 0 : paths->pref_addr = 0; /* Ignore the preferred address. */
1921 0 : goto out;
1922 : }
1923 : /* If the local address for new path is different from the current one, bind to
1924 : * the new address.
1925 : */
1926 28 : a = quic_path_saddr(paths, 1);
1927 28 : a->v4.sin_port = quic_path_saddr(paths, 0)->v4.sin_port;
1928 28 : if (!quic_cmp_sk_addr(sk, quic_path_saddr(paths, 0), a)) {
1929 19 : a->v4.sin_port = 0;
1930 19 : if (quic_path_bind(sk, paths, 1)) {
1931 0 : paths->pref_addr = 0; /* Ignore the preferred address. */
1932 0 : goto out;
1933 : }
1934 : }
1935 28 : goto out;
1936 : }
1937 :
1938 : /* Clean up transmitted handshake packets. */
1939 419 : quic_outq_transmitted_sack(sk, QUIC_CRYPTO_HANDSHAKE, QUIC_PN_MAP_MAX_PN, 0, -1, 0);
1940 419 : if (paths->pref_addr) {
1941 : /* If a preferred address is set, bind to it to allow client use at any time. */
1942 27 : err = quic_path_bind(sk, paths, 1);
1943 27 : if (err)
1944 : return err;
1945 : }
1946 :
1947 : /* Send NEW_TOKEN and HANDSHAKE_DONE frames (server only). */
1948 419 : if (quic_outq_transmit_frame(sk, QUIC_FRAME_NEW_TOKEN, NULL, 0, true))
1949 : return -ENOMEM;
1950 419 : if (quic_outq_transmit_frame(sk, QUIC_FRAME_HANDSHAKE_DONE, NULL, 0, true))
1951 : return -ENOMEM;
1952 419 : out:
1953 : /* Send NEW_CONNECTION_ID frames to ensure maximum connection IDs are added. */
1954 993 : if (quic_outq_transmit_new_conn_id(sk, 0, 0, false))
1955 : return -ENOMEM;
1956 : /* Enter established state, and start PLPMTUD timer and Path Challege timer. */
1957 993 : quic_set_state(sk, QUIC_SS_ESTABLISHED);
1958 993 : quic_timer_start(sk, QUIC_TIMER_PMTU, c->plpmtud_probe_interval);
1959 993 : quic_timer_reset_path(sk);
1960 993 : return 0;
1961 : }
1962 :
1963 6643 : static int quic_do_setsockopt(struct sock *sk, int optname, sockptr_t optval, unsigned int optlen)
1964 : {
1965 6643 : void *kopt = NULL;
1966 6643 : int retval = 0;
1967 :
1968 6643 : if (optlen > 0) {
1969 6607 : kopt = memdup_sockptr(optval, optlen);
1970 6607 : if (IS_ERR(kopt))
1971 0 : return PTR_ERR(kopt);
1972 : }
1973 :
1974 6643 : lock_sock(sk);
1975 6643 : switch (optname) {
1976 52 : case QUIC_SOCKOPT_EVENT:
1977 52 : retval = quic_sock_set_event(sk, kopt, optlen);
1978 52 : break;
1979 24 : case QUIC_SOCKOPT_STREAM_RESET:
1980 24 : retval = quic_sock_stream_reset(sk, kopt, optlen);
1981 24 : break;
1982 8 : case QUIC_SOCKOPT_STREAM_STOP_SENDING:
1983 8 : retval = quic_sock_stream_stop_sending(sk, kopt, optlen);
1984 8 : break;
1985 72 : case QUIC_SOCKOPT_CONNECTION_ID:
1986 72 : retval = quic_sock_set_connection_id(sk, kopt, optlen);
1987 72 : break;
1988 16 : case QUIC_SOCKOPT_CONNECTION_CLOSE:
1989 16 : retval = quic_sock_set_connection_close(sk, kopt, optlen);
1990 16 : break;
1991 43 : case QUIC_SOCKOPT_CONNECTION_MIGRATION:
1992 43 : retval = quic_sock_connection_migrate(sk, kopt, optlen);
1993 43 : break;
1994 : case QUIC_SOCKOPT_KEY_UPDATE:
1995 24 : retval = quic_crypto_key_update(quic_crypto(sk, QUIC_CRYPTO_APP));
1996 24 : break;
1997 639 : case QUIC_SOCKOPT_TRANSPORT_PARAM:
1998 639 : retval = quic_sock_set_transport_param(sk, kopt, optlen);
1999 639 : break;
2000 570 : case QUIC_SOCKOPT_CONFIG:
2001 570 : retval = quic_sock_set_config(sk, kopt, optlen);
2002 570 : break;
2003 126 : case QUIC_SOCKOPT_ALPN:
2004 126 : retval = quic_sock_set_alpn(sk, kopt, optlen);
2005 126 : break;
2006 12 : case QUIC_SOCKOPT_TOKEN:
2007 12 : retval = quic_sock_set_token(sk, kopt, optlen);
2008 12 : break;
2009 : case QUIC_SOCKOPT_SESSION_TICKET:
2010 4 : retval = quic_sock_set_session_ticket(sk, kopt, optlen);
2011 : break;
2012 994 : case QUIC_SOCKOPT_TRANSPORT_PARAM_EXT:
2013 994 : retval = quic_sock_set_transport_params_ext(sk, kopt, optlen);
2014 994 : break;
2015 4059 : case QUIC_SOCKOPT_CRYPTO_SECRET:
2016 4059 : retval = quic_sock_set_crypto_secret(sk, kopt, optlen);
2017 4059 : break;
2018 : default:
2019 : retval = -ENOPROTOOPT;
2020 : break;
2021 : }
2022 6643 : release_sock(sk);
2023 6643 : kfree(kopt);
2024 6643 : return retval;
2025 : }
2026 :
2027 6623 : static int quic_setsockopt(struct sock *sk, int level, int optname,
2028 : sockptr_t optval, unsigned int optlen)
2029 : {
2030 6623 : if (level != SOL_QUIC)
2031 0 : return quic_common_setsockopt(sk, level, optname, optval, optlen);
2032 :
2033 6623 : return quic_do_setsockopt(sk, optname, optval, optlen);
2034 : }
2035 :
2036 : /**
2037 : * quic_kernel_setsockopt - set a QUIC socket option from within the kernel
2038 : * @sk: socket to configure
2039 : * @optname: option name (QUIC-level)
2040 : * @optval: pointer to the option value
2041 : * @optlen: size of the option value
2042 : *
2043 : * Sets a QUIC socket option on a kernel socket without involving user space.
2044 : *
2045 : * Return values:
2046 : * - On success, 0 is returned.
2047 : * - On error, a negative error value is returned.
2048 : */
2049 20 : int quic_kernel_setsockopt(struct sock *sk, int optname, void *optval, unsigned int optlen)
2050 : {
2051 20 : return quic_do_setsockopt(sk, optname, KERNEL_SOCKPTR(optval), optlen);
2052 : }
2053 : EXPORT_SYMBOL_GPL(quic_kernel_setsockopt);
2054 :
2055 0 : static int quic_sock_get_event(struct sock *sk, u32 len, sockptr_t optval, sockptr_t optlen)
2056 : {
2057 0 : struct quic_inqueue *inq = quic_inq(sk);
2058 0 : struct quic_event_option event;
2059 :
2060 0 : if (len < sizeof(event))
2061 : return -EINVAL;
2062 0 : len = sizeof(event);
2063 0 : if (copy_from_sockptr(&event, optval, len))
2064 : return -EFAULT;
2065 :
2066 0 : if (!event.type || event.type > QUIC_EVENT_MAX)
2067 : return -EINVAL;
2068 : /* Set on if the corresponding event bit is set. */
2069 0 : event.on = !!(inq->events & BIT(event.type));
2070 :
2071 0 : if (copy_to_sockptr(optlen, &len, sizeof(len)) || copy_to_sockptr(optval, &event, len))
2072 0 : return -EFAULT;
2073 : return 0;
2074 : }
2075 :
2076 8830 : static int quic_sock_stream_open(struct sock *sk, u32 len, sockptr_t optval, sockptr_t optlen)
2077 : {
2078 8830 : struct quic_stream_table *streams = quic_streams(sk);
2079 8830 : struct quic_stream_info sinfo;
2080 8830 : struct quic_stream *stream;
2081 :
2082 8830 : if (len < sizeof(sinfo))
2083 : return -EINVAL;
2084 8830 : len = sizeof(sinfo);
2085 8830 : if (copy_from_sockptr(&sinfo, optval, len))
2086 : return -EFAULT;
2087 :
2088 8830 : if (sinfo.stream_flags & ~QUIC_MSG_STREAM_FLAGS) /* Reject unsupported flags. */
2089 : return -EINVAL;
2090 :
2091 : /* If stream_id is -1, assign the next available ID (bidi or uni). */
2092 8830 : if (sinfo.stream_id == -1) {
2093 144 : sinfo.stream_id = streams->send.next_bidi_stream_id;
2094 144 : if (sinfo.stream_flags & MSG_STREAM_UNI)
2095 112 : sinfo.stream_id = streams->send.next_uni_stream_id;
2096 : }
2097 8830 : sinfo.stream_flags |= MSG_STREAM_NEW; /* Mark stream as to be created. */
2098 :
2099 8830 : stream = quic_sock_send_stream(sk, &sinfo); /* Actually create or find the stream. */
2100 8830 : if (IS_ERR(stream))
2101 12 : return PTR_ERR(stream);
2102 :
2103 8818 : if (copy_to_sockptr(optlen, &len, sizeof(len)) || copy_to_sockptr(optval, &sinfo, len))
2104 0 : return -EFAULT;
2105 : return 0;
2106 : }
2107 :
2108 64 : static int quic_sock_get_connection_id(struct sock *sk, u32 len, sockptr_t optval, sockptr_t optlen)
2109 : {
2110 64 : struct quic_connection_id_info info;
2111 64 : struct quic_conn_id_set *id_set;
2112 64 : struct quic_conn_id *active;
2113 :
2114 64 : if (len < sizeof(info) || !quic_is_established(sk))
2115 : return -EINVAL;
2116 64 : len = sizeof(info);
2117 64 : if (copy_from_sockptr(&info, optval, len))
2118 : return -EFAULT;
2119 :
2120 64 : id_set = info.dest ? quic_dest(sk) : quic_source(sk);
2121 64 : active = quic_conn_id_active(id_set);
2122 64 : info.active = quic_conn_id_number(active);
2123 : /* Use prior_to to indicate the smallest issued connection ID number. Combined with
2124 : * the active_connection_id_limit (from the peer’s transport parameters), this allows
2125 : * userspace to infer the full set of valid connection IDs.
2126 : */
2127 64 : info.prior_to = quic_conn_id_first_number(id_set);
2128 :
2129 64 : if (copy_to_sockptr(optlen, &len, sizeof(len)) || copy_to_sockptr(optval, &info, len))
2130 0 : return -EFAULT;
2131 : return 0;
2132 : }
2133 :
2134 8 : static int quic_sock_get_connection_close(struct sock *sk, u32 len, sockptr_t optval,
2135 : sockptr_t optlen)
2136 : {
2137 8 : u8 *phrase, frame[QUIC_FRAME_BUF_LARGE] = {};
2138 8 : struct quic_outqueue *outq = quic_outq(sk);
2139 8 : struct quic_connection_close *close;
2140 8 : u32 phrase_len = 0;
2141 :
2142 8 : phrase = outq->close_phrase;
2143 8 : if (phrase)
2144 8 : phrase_len = strlen(phrase) + 1;
2145 8 : if (len < sizeof(*close) + phrase_len) /* Check if output buffer is large enough. */
2146 : return -EINVAL;
2147 :
2148 8 : len = sizeof(*close) + phrase_len;
2149 8 : close = (void *)frame;
2150 8 : close->errcode = outq->close_errcode;
2151 8 : close->frame = outq->close_frame;
2152 :
2153 8 : if (phrase_len)
2154 4 : strscpy(close->phrase, phrase, phrase_len);
2155 :
2156 8 : if (copy_to_sockptr(optlen, &len, sizeof(len)) || copy_to_sockptr(optval, close, len))
2157 0 : return -EFAULT;
2158 : return 0;
2159 : }
2160 :
2161 19 : static int quic_sock_get_transport_param(struct sock *sk, u32 len,
2162 : sockptr_t optval, sockptr_t optlen)
2163 : {
2164 19 : struct quic_transport_param param = {};
2165 :
2166 19 : if (len < sizeof(param))
2167 : return -EINVAL;
2168 19 : len = sizeof(param);
2169 19 : if (copy_from_sockptr(¶m, optval, len))
2170 : return -EFAULT;
2171 :
2172 19 : quic_sock_fetch_transport_param(sk, ¶m);
2173 :
2174 19 : if (copy_to_sockptr(optlen, &len, sizeof(len)) || copy_to_sockptr(optval, ¶m, len))
2175 0 : return -EFAULT;
2176 : return 0;
2177 : }
2178 :
2179 16 : static int quic_sock_get_config(struct sock *sk, u32 len, sockptr_t optval, sockptr_t optlen)
2180 : {
2181 16 : struct quic_config config, *c = quic_config(sk);
2182 :
2183 16 : if (len < sizeof(config))
2184 : return -EINVAL;
2185 16 : len = sizeof(config);
2186 :
2187 16 : config = *c;
2188 16 : if (copy_to_sockptr(optlen, &len, sizeof(len)) || copy_to_sockptr(optval, &config, len))
2189 0 : return -EFAULT;
2190 : return 0;
2191 : }
2192 :
2193 16 : static int quic_sock_get_alpn(struct sock *sk, u32 len, sockptr_t optval, sockptr_t optlen)
2194 : {
2195 16 : struct quic_data *alpns = quic_alpn(sk);
2196 16 : u8 data[QUIC_ALPN_MAX_LEN];
2197 :
2198 16 : if (!alpns->len) {
2199 0 : len = 0;
2200 0 : goto out;
2201 : }
2202 16 : if (len < alpns->len)
2203 : return -EINVAL;
2204 :
2205 16 : quic_data_to_string(data, &len, alpns);
2206 :
2207 16 : out:
2208 16 : if (copy_to_sockptr(optlen, &len, sizeof(len)) || copy_to_sockptr(optval, data, len))
2209 0 : return -EFAULT;
2210 : return 0;
2211 : }
2212 :
2213 989 : static int quic_sock_get_token(struct sock *sk, u32 len, sockptr_t optval, sockptr_t optlen)
2214 : {
2215 989 : struct quic_data *token = quic_token(sk);
2216 :
2217 989 : if (quic_is_serv(sk) || len < token->len)
2218 : return -EINVAL;
2219 577 : len = token->len;
2220 :
2221 577 : if (copy_to_sockptr(optlen, &len, sizeof(len)) || copy_to_sockptr(optval, token->data, len))
2222 0 : return -EFAULT;
2223 : return 0;
2224 : }
2225 :
2226 : #define QUIC_TICKET_MASTER_KEY_LEN 64
2227 :
2228 420 : static int quic_sock_get_session_ticket(struct sock *sk, u32 len,
2229 : sockptr_t optval, sockptr_t optlen)
2230 : {
2231 420 : u8 *ticket = quic_ticket(sk)->data, key[QUIC_TICKET_MASTER_KEY_LEN];
2232 420 : struct quic_crypto *crypto = quic_crypto(sk, QUIC_CRYPTO_APP);
2233 420 : u32 tlen = quic_ticket(sk)->len;
2234 420 : union quic_addr a;
2235 :
2236 420 : if (!quic_is_serv(sk)) {
2237 : /* For clients, retrieve the received TLS NewSessionTicket message. */
2238 50 : if (quic_is_established(sk) && !crypto->ticket_ready)
2239 18 : tlen = 0;
2240 50 : goto out;
2241 : }
2242 :
2243 370 : if (quic_is_closed(sk))
2244 : return -EINVAL;
2245 :
2246 : /* For servers, return the master key used for session resumption. If already set,
2247 : * reuse it.
2248 : */
2249 370 : if (tlen)
2250 0 : goto out;
2251 :
2252 : /* If not already set, derive the key using the peer address. */
2253 370 : crypto = quic_crypto(sk, QUIC_CRYPTO_INITIAL);
2254 370 : memcpy(&a, quic_path_daddr(quic_paths(sk), 0), sizeof(a));
2255 370 : a.v4.sin_port = 0;
2256 370 : if (quic_crypto_generate_session_ticket_key(crypto, &a, sizeof(a), key,
2257 : QUIC_TICKET_MASTER_KEY_LEN))
2258 : return -EINVAL;
2259 : ticket = key;
2260 : tlen = QUIC_TICKET_MASTER_KEY_LEN;
2261 420 : out:
2262 420 : if (len < tlen)
2263 : return -EINVAL;
2264 420 : len = tlen;
2265 :
2266 420 : if (copy_to_sockptr(optlen, &len, sizeof(len)) || copy_to_sockptr(optval, ticket, len))
2267 0 : return -EFAULT;
2268 : return 0;
2269 : }
2270 :
2271 1001 : static int quic_sock_get_transport_params_ext(struct sock *sk, u32 len,
2272 : sockptr_t optval, sockptr_t optlen)
2273 : {
2274 1001 : struct quic_transport_param param = {};
2275 1001 : u8 data[QUIC_TP_EXT_MAX_LEN];
2276 1001 : u32 datalen = 0;
2277 :
2278 1001 : if (!quic_is_establishing(sk))
2279 : return -EINVAL;
2280 :
2281 1001 : quic_sock_fetch_transport_param(sk, ¶m);
2282 :
2283 1001 : if (quic_frame_build_transport_params_ext(sk, ¶m, data, &datalen))
2284 : return -EINVAL;
2285 1001 : if (len < datalen)
2286 : return -EINVAL;
2287 1001 : len = datalen;
2288 :
2289 1001 : if (copy_to_sockptr(optlen, &len, sizeof(len)) || copy_to_sockptr(optval, data, len))
2290 0 : return -EFAULT;
2291 : return 0;
2292 : }
2293 :
2294 0 : static int quic_sock_get_crypto_secret(struct sock *sk, u32 len,
2295 : sockptr_t optval, sockptr_t optlen)
2296 : {
2297 0 : struct quic_crypto_secret secret = {};
2298 :
2299 0 : if (len < sizeof(secret))
2300 : return -EINVAL;
2301 0 : len = sizeof(secret);
2302 0 : if (copy_from_sockptr(&secret, optval, len))
2303 : return -EFAULT;
2304 :
2305 0 : if (secret.level >= QUIC_CRYPTO_MAX)
2306 : return -EINVAL;
2307 0 : if (quic_crypto_get_secret(quic_crypto(sk, secret.level), &secret))
2308 : return -EINVAL;
2309 :
2310 0 : if (copy_to_sockptr(optlen, &len, sizeof(len)) || copy_to_sockptr(optval, &secret, len))
2311 0 : return -EFAULT;
2312 : return 0;
2313 : }
2314 :
2315 11363 : static int quic_do_getsockopt(struct sock *sk, int optname, sockptr_t optval, sockptr_t optlen)
2316 : {
2317 11363 : int retval = 0;
2318 11363 : u32 len;
2319 :
2320 11363 : if (copy_from_sockptr(&len, optlen, sizeof(len)))
2321 : return -EFAULT;
2322 :
2323 11363 : lock_sock(sk);
2324 11363 : switch (optname) {
2325 0 : case QUIC_SOCKOPT_EVENT:
2326 0 : retval = quic_sock_get_event(sk, len, optval, optlen);
2327 0 : break;
2328 8830 : case QUIC_SOCKOPT_STREAM_OPEN:
2329 8830 : retval = quic_sock_stream_open(sk, len, optval, optlen);
2330 8830 : break;
2331 64 : case QUIC_SOCKOPT_CONNECTION_ID:
2332 64 : retval = quic_sock_get_connection_id(sk, len, optval, optlen);
2333 64 : break;
2334 8 : case QUIC_SOCKOPT_CONNECTION_CLOSE:
2335 8 : retval = quic_sock_get_connection_close(sk, len, optval, optlen);
2336 8 : break;
2337 19 : case QUIC_SOCKOPT_TRANSPORT_PARAM:
2338 19 : retval = quic_sock_get_transport_param(sk, len, optval, optlen);
2339 19 : break;
2340 16 : case QUIC_SOCKOPT_CONFIG:
2341 16 : retval = quic_sock_get_config(sk, len, optval, optlen);
2342 16 : break;
2343 16 : case QUIC_SOCKOPT_ALPN:
2344 16 : retval = quic_sock_get_alpn(sk, len, optval, optlen);
2345 16 : break;
2346 989 : case QUIC_SOCKOPT_TOKEN:
2347 989 : retval = quic_sock_get_token(sk, len, optval, optlen);
2348 989 : break;
2349 420 : case QUIC_SOCKOPT_SESSION_TICKET:
2350 420 : retval = quic_sock_get_session_ticket(sk, len, optval, optlen);
2351 420 : break;
2352 1001 : case QUIC_SOCKOPT_TRANSPORT_PARAM_EXT:
2353 1001 : retval = quic_sock_get_transport_params_ext(sk, len, optval, optlen);
2354 1001 : break;
2355 0 : case QUIC_SOCKOPT_CRYPTO_SECRET:
2356 0 : retval = quic_sock_get_crypto_secret(sk, len, optval, optlen);
2357 0 : break;
2358 : default:
2359 : retval = -ENOPROTOOPT;
2360 : break;
2361 : }
2362 11363 : release_sock(sk);
2363 11363 : return retval;
2364 : }
2365 :
2366 11359 : static int quic_getsockopt(struct sock *sk, int level, int optname,
2367 : char __user *optval, int __user *optlen)
2368 : {
2369 11359 : if (level != SOL_QUIC)
2370 0 : return quic_common_getsockopt(sk, level, optname, optval, optlen);
2371 :
2372 11359 : return quic_do_getsockopt(sk, optname, USER_SOCKPTR(optval), USER_SOCKPTR(optlen));
2373 : }
2374 :
2375 : /**
2376 : * quic_kernel_getsockopt - get a QUIC socket option from within the kernel
2377 : * @sk: socket to query
2378 : * @optname: option name (QUIC-level)
2379 : * @optval: pointer to the buffer to receive the option value
2380 : * @optlen: pointer to the size of the buffer; updated to actual length on return
2381 : *
2382 : * Gets a QUIC socket option from a kernel socket, bypassing user space.
2383 : *
2384 : * Return values:
2385 : * - On success, 0 is returned.
2386 : * - On error, a negative error value is returned.
2387 : */
2388 4 : int quic_kernel_getsockopt(struct sock *sk, int optname, void *optval, unsigned int *optlen)
2389 : {
2390 4 : return quic_do_getsockopt(sk, optname, KERNEL_SOCKPTR(optval), KERNEL_SOCKPTR(optlen));
2391 : }
2392 : EXPORT_SYMBOL_GPL(quic_kernel_getsockopt);
2393 :
2394 2852405 : static void quic_release_cb(struct sock *sk)
2395 : {
2396 : /* Similar to tcp_release_cb(). */
2397 2852405 : unsigned long nflags, flags = smp_load_acquire(&sk->sk_tsq_flags);
2398 :
2399 2852407 : do {
2400 2852407 : if (!(flags & QUIC_DEFERRED_ALL))
2401 2852407 : return;
2402 1249 : nflags = flags & ~QUIC_DEFERRED_ALL;
2403 1249 : } while (!try_cmpxchg(&sk->sk_tsq_flags, &flags, nflags));
2404 :
2405 1249 : if (flags & QUIC_F_MTU_REDUCED_DEFERRED) {
2406 488 : quic_packet_rcv_err_pmtu(sk);
2407 488 : __sock_put(sk);
2408 : }
2409 1249 : if (flags & QUIC_F_LOSS_DEFERRED) {
2410 0 : quic_timer_loss_handler(sk);
2411 0 : __sock_put(sk);
2412 : }
2413 1249 : if (flags & QUIC_F_SACK_DEFERRED) {
2414 595 : quic_timer_sack_handler(sk);
2415 595 : __sock_put(sk);
2416 : }
2417 1249 : if (flags & QUIC_F_PATH_DEFERRED) {
2418 0 : quic_timer_path_handler(sk);
2419 0 : __sock_put(sk);
2420 : }
2421 1249 : if (flags & QUIC_F_PMTU_DEFERRED) {
2422 1 : quic_timer_pmtu_handler(sk);
2423 1 : __sock_put(sk);
2424 : }
2425 1249 : if (flags & QUIC_F_TSQ_DEFERRED) {
2426 165 : quic_timer_pace_handler(sk);
2427 165 : __sock_put(sk);
2428 : }
2429 : }
2430 :
2431 0 : static int quic_disconnect(struct sock *sk, int flags)
2432 : {
2433 0 : quic_set_state(sk, QUIC_SS_CLOSED); /* for a listen socket only */
2434 0 : return 0;
2435 : }
2436 :
2437 4 : static void quic_shutdown(struct sock *sk, int how)
2438 : {
2439 4 : if (!(how & SEND_SHUTDOWN))
2440 0 : goto out;
2441 :
2442 4 : quic_outq_transmit_app_close(sk);
2443 4 : out:
2444 4 : quic_set_state(sk, QUIC_SS_CLOSED);
2445 4 : }
2446 :
2447 : struct proto quic_prot = {
2448 : .name = "QUIC",
2449 : .owner = THIS_MODULE,
2450 : .ioctl = quic_ioctl,
2451 : .init = quic_init_sock,
2452 : .destroy = quic_destroy_sock,
2453 : .shutdown = quic_shutdown,
2454 : .setsockopt = quic_setsockopt,
2455 : .getsockopt = quic_getsockopt,
2456 : .connect = quic_connect,
2457 : .bind = quic_bind,
2458 : .close = quic_close,
2459 : .disconnect = quic_disconnect,
2460 : .sendmsg = quic_sendmsg,
2461 : .recvmsg = quic_recvmsg,
2462 : .accept = quic_accept,
2463 : .hash = quic_hash,
2464 : .unhash = quic_unhash,
2465 : .backlog_rcv = quic_packet_process,
2466 : .release_cb = quic_release_cb,
2467 : .no_autobind = true,
2468 : .obj_size = sizeof(struct quic_sock),
2469 : .sysctl_mem = sysctl_quic_mem,
2470 : .sysctl_rmem = sysctl_quic_rmem,
2471 : .sysctl_wmem = sysctl_quic_wmem,
2472 : .memory_pressure = &quic_memory_pressure,
2473 : .enter_memory_pressure = quic_enter_memory_pressure,
2474 : .memory_allocated = &quic_memory_allocated,
2475 : .per_cpu_fw_alloc = &quic_memory_per_cpu_fw_alloc,
2476 : .sockets_allocated = &quic_sockets_allocated,
2477 : };
2478 :
2479 : struct proto quicv6_prot = {
2480 : .name = "QUICv6",
2481 : .owner = THIS_MODULE,
2482 : .ioctl = quic_ioctl,
2483 : .init = quic_init_sock,
2484 : .destroy = quic_destroy_sock,
2485 : .shutdown = quic_shutdown,
2486 : .setsockopt = quic_setsockopt,
2487 : .getsockopt = quic_getsockopt,
2488 : .connect = quic_connect,
2489 : .bind = quic_bind,
2490 : .close = quic_close,
2491 : .disconnect = quic_disconnect,
2492 : .sendmsg = quic_sendmsg,
2493 : .recvmsg = quic_recvmsg,
2494 : .accept = quic_accept,
2495 : .hash = quic_hash,
2496 : .unhash = quic_unhash,
2497 : .backlog_rcv = quic_packet_process,
2498 : .release_cb = quic_release_cb,
2499 : .no_autobind = true,
2500 : .obj_size = sizeof(struct quic6_sock),
2501 : .ipv6_pinfo_offset = offsetof(struct quic6_sock, inet6),
2502 : .sysctl_mem = sysctl_quic_mem,
2503 : .sysctl_rmem = sysctl_quic_rmem,
2504 : .sysctl_wmem = sysctl_quic_wmem,
2505 : .memory_pressure = &quic_memory_pressure,
2506 : .enter_memory_pressure = quic_enter_memory_pressure,
2507 : .memory_allocated = &quic_memory_allocated,
2508 : .per_cpu_fw_alloc = &quic_memory_per_cpu_fw_alloc,
2509 : .sockets_allocated = &quic_sockets_allocated,
2510 : };
|
