Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0-or-later
2 : /* QUIC kernel implementation
3 : * (C) Copyright Red Hat Corp. 2023
4 : *
5 : * This file is part of the QUIC kernel implementation
6 : *
7 : * Initialization/cleanup for QUIC protocol support.
8 : *
9 : * Written or modified by:
10 : * Xin Long <lucien.xin@gmail.com>
11 : */
12 :
13 : #include "socket.h"
14 :
15 : /* Checks whether a frame can be transmmited based on congestion control and Anti-Amplification. */
16 11349901 : static int quic_outq_limit_check(struct sock *sk, struct quic_frame *frame)
17 : {
18 11349901 : struct quic_path_group *paths = quic_paths(sk);
19 11349901 : struct quic_packet *packet = quic_packet(sk);
20 11349901 : struct quic_outqueue *outq = quic_outq(sk);
21 11349901 : u16 len;
22 :
23 : /* If in single-packet mode, allow only one packet to transmit. */
24 11349901 : if (outq->single && outq->count)
25 : return -1;
26 :
27 : /* Enforce congestion control for ack-eliciting frames except PING. */
28 11349165 : if (!outq->single && frame->ack_eliciting && !quic_frame_ping(frame->type)) {
29 10949920 : len = packet->frame_len + frame->len;
30 10949920 : if (outq->inflight + len > outq->window)
31 : return -1;
32 : }
33 :
34 : /* rfc9000#section-21.1.1.1: Anti-amplification limit for server before path validation. */
35 11237855 : if (quic_is_serv(sk) && !paths->validated) {
36 3866 : len = packet->len + frame->len + quic_packet_taglen(packet);
37 3866 : if (paths->ampl_sndlen + len > paths->ampl_rcvlen * 3) {
38 35 : paths->blocked = 1;
39 35 : return -1;
40 : }
41 : }
42 :
43 : return 0;
44 : }
45 :
46 : /* Flush any appeneded frames or coalesced/bundled packets. */
47 13186154 : static int quic_outq_transmit_flush(struct sock *sk)
48 : {
49 13186154 : struct quic_packet *packet = quic_packet(sk);
50 13186154 : struct quic_outqueue *outq = quic_outq(sk);
51 13186154 : int count = outq->count;
52 :
53 13186154 : outq->count = 0;
54 13186154 : if (!quic_packet_empty(packet))
55 4507251 : count += quic_packet_create(sk);
56 13189473 : quic_packet_flush(sk);
57 :
58 13179033 : return count;
59 : }
60 :
61 : /* Transmits control frames at a given encryption level (Initial, Handshake, 1-RTT)). */
62 39500417 : static void quic_outq_transmit_ctrl(struct sock *sk, u8 level)
63 : {
64 39500417 : struct quic_pnspace *space = quic_pnspace(sk, level);
65 39500417 : struct quic_outqueue *outq = quic_outq(sk);
66 39500417 : struct quic_frame *frame, *next;
67 39500417 : struct list_head *head;
68 :
69 39500417 : if (!quic_crypto(sk, level)->send_ready)
70 : return;
71 :
72 39497631 : if (space->need_sack) { /* Transmit SACK (ACK for crypto space) first if needed. */
73 378481 : if (!quic_outq_transmit_frame(sk, QUIC_FRAME_ACK, &level,
74 378481 : space->sack_path, true))
75 378481 : space->need_sack = 0;
76 : }
77 :
78 39497631 : head = &outq->control_list;
79 40066783 : list_for_each_entry_safe(frame, next, head, list) {
80 876643 : if (!frame->level && level) /* Initial, Handshake levels precede 1-RTT (0). */
81 : break;
82 554206 : if (frame->level != level)
83 2304 : continue;
84 551902 : if (quic_packet_config(sk, frame->level, frame->path))
85 : break;
86 566854 : if (quic_outq_limit_check(sk, frame))
87 : break;
88 566849 : if (quic_packet_tail(sk, frame))
89 564641 : continue; /* Frame appeneded. */
90 : /* Flush already appended frames before processing this one. */
91 2207 : outq->count += quic_packet_create(sk);
92 2207 : next = frame; /* Re-append this frame. */
93 : }
94 : }
95 :
96 : /* Transmit application datagrams (QUIC DATAGRAM frames). */
97 13177745 : static void quic_outq_transmit_dgram(struct sock *sk)
98 : {
99 13177745 : struct quic_outqueue *outq = quic_outq(sk);
100 13177745 : struct quic_frame *frame, *next;
101 13177745 : struct list_head *head;
102 :
103 13177745 : if (!quic_crypto(sk, outq->data_level)->send_ready)
104 : return;
105 :
106 13176277 : head = &outq->datagram_list;
107 13176285 : list_for_each_entry_safe(frame, next, head, list) {
108 0 : if (quic_packet_config(sk, outq->data_level, frame->path))
109 : break;
110 8 : if (quic_outq_limit_check(sk, frame))
111 : break;
112 8 : if (quic_packet_tail(sk, frame))
113 8 : continue;
114 0 : outq->count += quic_packet_create(sk);
115 0 : next = frame;
116 : }
117 : }
118 :
119 : /* Applies stream and connection-level flow control. Returns 1 if blocked, 0 otherwise.
120 : * Send a STREAM_DATA_BLOCKED or DATA_BLOCKED frame if blocked and sndblock is set.
121 : */
122 8310934 : int quic_outq_flow_control(struct sock *sk, struct quic_stream *stream, u16 bytes, u8 sndblock)
123 : {
124 8310934 : struct quic_outqueue *outq = quic_outq(sk);
125 8310934 : u8 frame, blocked = 0, transmit = 0;
126 :
127 : /* Check stream-level flow control. */
128 8310934 : if (stream->send.bytes + bytes > stream->send.max_bytes) {
129 : /* Send a STREAM_DATA_BLOCKED frame only after the previous one is acknowledged,
130 : * and stream->send.max_bytes has been updated via a received MAX_STREAM_DATA frame.
131 : */
132 15295 : if (!stream->send.data_blocked &&
133 31 : stream->send.last_max_bytes < stream->send.max_bytes) {
134 31 : frame = QUIC_FRAME_STREAM_DATA_BLOCKED;
135 31 : if (sndblock && !quic_outq_transmit_frame(sk, frame, stream, 0, true))
136 0 : transmit = 1;
137 31 : stream->send.last_max_bytes = stream->send.max_bytes;
138 31 : stream->send.data_blocked = 1;
139 : }
140 : blocked = 1;
141 : }
142 : /* Check connection-level flow control. */
143 8310934 : if (outq->bytes + bytes > outq->max_bytes) {
144 : /* Send a DATA_BLOCKED frame only after the previous one is acknowledged,
145 : * and max_bytes has been updated via a received MAX_STREAM_DATA frame.
146 : */
147 2 : if (!outq->data_blocked && outq->last_max_bytes < outq->max_bytes) {
148 1 : frame = QUIC_FRAME_DATA_BLOCKED;
149 1 : if (sndblock && !quic_outq_transmit_frame(sk, frame, outq, 0, true))
150 0 : transmit = 1;
151 1 : outq->last_max_bytes = outq->max_bytes;
152 1 : outq->data_blocked = 1;
153 : }
154 : blocked = 1;
155 : }
156 :
157 8310934 : if (transmit)
158 0 : quic_outq_transmit(sk);
159 :
160 8310934 : return blocked;
161 : }
162 :
163 : /* Returns available writable space considering stream limits if stream is set,
164 : * otherwise connection limits.
165 : */
166 10536199 : u64 quic_outq_wspace(struct sock *sk, struct quic_stream *stream)
167 : {
168 10536199 : struct quic_outqueue *outq = quic_outq(sk);
169 10536199 : u64 len = outq->max_bytes - outq->bytes;
170 :
171 10536199 : if (stream) {
172 10253667 : len = min_t(u64, len, sk_stream_wspace(sk));
173 10253667 : len = min_t(u64, len, stream->send.max_bytes - stream->send.bytes);
174 : }
175 :
176 10536199 : return len;
177 : }
178 :
179 : /* Applies pacing and Nagle’s algorithm. Returns 1 if sending should be delayed, 0 if
180 : * immediate send.
181 : */
182 10666126 : static int quic_outq_delay_check(struct sock *sk, u8 level, u8 nodelay)
183 : {
184 10666126 : struct quic_packet *packet = quic_packet(sk);
185 10666126 : struct quic_outqueue *outq = quic_outq(sk);
186 10666126 : u64 pacing_time;
187 :
188 10666126 : if (level || outq->close_frame) /* No delay for early data or if connection is closing. */
189 : return 0;
190 :
191 10665923 : pacing_time = quic_cong(sk)->pacing_time;
192 10665923 : if (pacing_time > ktime_get_ns()) { /* Delay data transmission in PACE timer. */
193 172243 : quic_timer_start(sk, QUIC_TIMER_PACE, pacing_time);
194 172243 : return 1;
195 : }
196 :
197 10493680 : if (nodelay) /* If this frame the frame is not the last of a sendmsg. */
198 : return 0;
199 : /* If there’s already data queued in the packet, send immediately. */
200 2279159 : if (!quic_packet_empty(packet))
201 : return 0;
202 : /* If Nagle is disabled via config or no data is in flight, and MSG_MORE isn't set,
203 : * allow immediate send.
204 : */
205 2111271 : if ((quic_config(sk)->stream_data_nodelay || !outq->inflight) &&
206 719 : !outq->force_delay)
207 : return 0;
208 : /* If enough stream data is available to build a full-sized packet, send immediately. */
209 2110552 : if (outq->stream_list_len > quic_packet_mss(packet))
210 97 : return 0;
211 : return 1; /* Otherwise, delay sending to coalesce more data. */
212 : }
213 :
214 : /* Sends stream data frames. */
215 13178653 : static void quic_outq_transmit_stream(struct sock *sk)
216 : {
217 13178653 : struct quic_outqueue *outq = quic_outq(sk);
218 13178653 : struct quic_frame *frame, *next;
219 13178653 : struct list_head *head;
220 :
221 : /* Although frame->level is always App, stream data may need to be sent at App or Early
222 : * level depending on key availability. Use outq->data_level to select the level.
223 : */
224 13178653 : if (!quic_crypto(sk, outq->data_level)->send_ready)
225 : return;
226 :
227 13177185 : head = &outq->stream_list;
228 21560613 : list_for_each_entry_safe(frame, next, head, list) {
229 10775466 : if (quic_packet_config(sk, outq->data_level, frame->path))
230 : break;
231 10777733 : if (quic_outq_limit_check(sk, frame))
232 : break;
233 10666126 : if (quic_outq_delay_check(sk, outq->data_level, frame->nodelay))
234 : break;
235 8383428 : if (quic_packet_tail(sk, frame)) {
236 6337622 : outq->stream_list_len -= frame->len;
237 6337622 : continue;
238 : }
239 2045806 : outq->count += quic_packet_create(sk);
240 2045806 : next = frame;
241 : }
242 : }
243 :
244 : /* Transmits pending frames at a specific encryption level from transmitted_list. */
245 2387 : static void quic_outq_transmit_old(struct sock *sk, u8 level)
246 : {
247 2387 : struct quic_outqueue *outq = quic_outq(sk);
248 2387 : struct quic_frame *frame, *next;
249 2387 : struct list_head *head;
250 :
251 2387 : head = &outq->transmitted_list;
252 7919 : list_for_each_entry_safe(frame, next, head, list) {
253 6227 : if (!frame->level && level)
254 : break;
255 5998 : if (frame->level != level)
256 686 : continue;
257 : /* Do not transmit old Datagram and PING frames. */
258 5312 : if (quic_frame_dgram(frame->type) || quic_frame_ping(frame->type))
259 0 : continue;
260 5312 : if (!quic_crypto(sk, frame->level)->send_ready)
261 : break;
262 5312 : if (quic_packet_config(sk, frame->level, frame->path))
263 : break;
264 5310 : if (quic_outq_limit_check(sk, frame))
265 : break;
266 4844 : if (quic_packet_tail(sk, frame))
267 4708 : continue;
268 138 : outq->count += quic_packet_create(sk);
269 138 : next = frame;
270 : }
271 2389 : }
272 :
273 : /* Sends all pending frames from the outqueue. Returns number of packets sent. */
274 13176682 : int quic_outq_transmit(struct sock *sk)
275 : {
276 13176682 : quic_outq_transmit_ctrl(sk, QUIC_CRYPTO_INITIAL);
277 13174912 : quic_outq_transmit_ctrl(sk, QUIC_CRYPTO_HANDSHAKE);
278 13174334 : quic_outq_transmit_ctrl(sk, QUIC_CRYPTO_APP);
279 :
280 13176018 : quic_outq_transmit_dgram(sk);
281 13177460 : quic_outq_transmit_stream(sk);
282 :
283 13169134 : return quic_outq_transmit_flush(sk);
284 : }
285 :
286 : /* Transmits at most one packet at the specified encryption level. */
287 2386 : static int quic_outq_transmit_single(struct sock *sk, u8 level)
288 : {
289 2386 : struct quic_outqueue *outq = quic_outq(sk);
290 :
291 2386 : outq->single = 1; /* Mark that we are doing a single-packet transmission. */
292 2386 : quic_outq_transmit_ctrl(sk, level);
293 :
294 2387 : if (level == QUIC_CRYPTO_APP) {
295 : /* Only need to transmit DATAGRAM and STREAM frames at application level. */
296 1305 : quic_outq_transmit_dgram(sk);
297 1305 : quic_outq_transmit_stream(sk);
298 : }
299 :
300 : /* Try sending frames in transmitted_list if no new frame was packed. */
301 2387 : quic_outq_transmit_old(sk, level);
302 2389 : outq->single = 0;
303 :
304 2389 : return quic_outq_transmit_flush(sk);
305 : }
306 :
307 : /* Frees socket memory resources after send. */
308 6489339 : static void quic_outq_wfree(int len, struct sock *sk)
309 : {
310 6489339 : if (!len)
311 : return;
312 :
313 6317394 : WARN_ON(refcount_sub_and_test(len, &sk->sk_wmem_alloc));
314 6317394 : sk_wmem_queued_add(sk, -len);
315 6317394 : sk_mem_uncharge(sk, len);
316 :
317 6317394 : if (sk_stream_wspace(sk) > 0)
318 6190252 : sk->sk_write_space(sk);
319 : }
320 :
321 : /* Charges memory to socket for new frame. */
322 8844298 : static void quic_outq_set_owner_w(int len, struct sock *sk)
323 : {
324 8844298 : if (!len)
325 : return;
326 :
327 8295330 : refcount_add(len, &sk->sk_wmem_alloc);
328 8295330 : sk_wmem_queued_add(sk, len);
329 8295330 : sk_mem_charge(sk, len);
330 : }
331 :
332 : /* Appends data to an existing stream frame at the tail of the stream_list if possible. */
333 10253667 : int quic_outq_stream_append(struct sock *sk, struct quic_msginfo *info, u8 pack)
334 : {
335 10253667 : struct quic_stream_table *streams = quic_streams(sk);
336 10253667 : struct quic_outqueue *outq = quic_outq(sk);
337 10253667 : struct quic_stream *stream = info->stream;
338 10253667 : struct quic_frame *frame;
339 10253667 : struct list_head *head;
340 10253667 : int len, bytes;
341 :
342 10253667 : head = &outq->stream_list;
343 10253667 : if (list_empty(head))
344 : return -1;
345 : /* Append only if it's the same stream, the frame is the last of a sendmsg (i.e.,
346 : * !nodelay) and it hasn't been transmitted yet (offset < 0).
347 : */
348 6022378 : frame = list_last_entry(head, struct quic_frame, list);
349 6022378 : if (frame->stream != stream || frame->nodelay || frame->offset >= 0)
350 : return -1;
351 :
352 3923667 : len = frame->len;
353 3923667 : bytes = quic_frame_stream_append(sk, frame, info, pack);
354 : /* If append failed or this was just a size probe, return immediately. */
355 3923667 : if (bytes < 0 || !pack)
356 : return bytes;
357 :
358 : /* If FIN bit is now set and the stream was in SEND state, mark it SENT and clear
359 : * active_stream_id if it matches.
360 : */
361 1961831 : if (frame->type & QUIC_STREAM_BIT_FIN &&
362 9 : stream->send.state == QUIC_STREAM_SEND_STATE_SEND) {
363 9 : if (streams->send.active_stream_id == stream->id)
364 8 : streams->send.active_stream_id = -1;
365 9 : stream->send.state = QUIC_STREAM_SEND_STATE_SENT;
366 : }
367 :
368 : /* Update accounting. */
369 1961831 : stream->send.bytes += bytes;
370 :
371 1961831 : outq->bytes += bytes;
372 1961831 : outq->stream_list_len += (frame->len - len);
373 1961831 : outq->unsent_bytes += bytes;
374 1961831 : quic_outq_set_owner_w((int)bytes, sk);
375 :
376 1961831 : return bytes;
377 : }
378 :
379 : /* Queues a stream frame at the tail of the stream list and optionally triggers transmission. */
380 6330005 : void quic_outq_stream_tail(struct sock *sk, struct quic_frame *frame, bool cork)
381 : {
382 6330005 : struct quic_stream_table *streams = quic_streams(sk);
383 6330005 : struct quic_stream *stream = frame->stream;
384 6330005 : struct quic_outqueue *outq = quic_outq(sk);
385 :
386 : /* rfc9000#section-3.1:
387 : *
388 : * Sending the first STREAM or STREAM_DATA_BLOCKED frame causes a sending part of a
389 : * stream to enter the "Send" state.
390 : *
391 : * After the application indicates that all stream data has been sent and a STREAM
392 : * frame containing the FIN bit is sent, the sending part of the stream enters the
393 : * "Data Sent" state.
394 : */
395 6330005 : if (stream->send.state == QUIC_STREAM_SEND_STATE_READY)
396 17760 : stream->send.state = QUIC_STREAM_SEND_STATE_SEND;
397 :
398 6330005 : if (frame->type & QUIC_STREAM_BIT_FIN &&
399 17623 : stream->send.state == QUIC_STREAM_SEND_STATE_SEND) {
400 : /* Clear active_stream_id if it matches the finished stream. */
401 17623 : if (streams->send.active_stream_id == stream->id)
402 9677 : streams->send.active_stream_id = -1;
403 17623 : stream->send.state = QUIC_STREAM_SEND_STATE_SENT;
404 : }
405 :
406 : /* Update accounting. */
407 6330005 : stream->send.frags++;
408 6330005 : stream->send.bytes += frame->bytes;
409 :
410 6330005 : outq->bytes += frame->bytes;
411 6330005 : outq->stream_list_len += frame->len;
412 6330005 : outq->unsent_bytes += frame->bytes;
413 6330005 : quic_outq_set_owner_w((int)frame->bytes, sk);
414 :
415 6330005 : list_add_tail(&frame->list, &outq->stream_list);
416 6330005 : if (!cork) /* If not corked, trigger transmission immediately. */
417 6330005 : quic_outq_transmit(sk);
418 6330005 : }
419 :
420 : /* Queues a datagram frame at the tail of the datagram list and optionally transmits. */
421 8 : void quic_outq_dgram_tail(struct sock *sk, struct quic_frame *frame, bool cork)
422 : {
423 8 : struct quic_outqueue *outq = quic_outq(sk);
424 :
425 8 : outq->unsent_bytes += frame->bytes;
426 8 : quic_outq_set_owner_w((int)frame->bytes, sk);
427 8 : list_add_tail(&frame->list, &outq->datagram_list);
428 8 : if (!cork)
429 8 : quic_outq_transmit(sk);
430 8 : }
431 :
432 : /* Queues a control frame in control_list in correct order and optionally transmits. */
433 563496 : void quic_outq_ctrl_tail(struct sock *sk, struct quic_frame *frame, bool cork)
434 : {
435 563496 : struct quic_outqueue *outq = quic_outq(sk);
436 563496 : struct list_head *head;
437 563496 : struct quic_frame *pos;
438 :
439 563496 : head = &quic_outq(sk)->control_list;
440 : /* Insert control frame in priority order:
441 : *
442 : * Handshake levels (level > 0) > Non-ack-eliciting > Other frames.
443 : */
444 567278 : list_for_each_entry(pos, head, list) {
445 183004 : if (frame->level) {
446 5383 : if (!pos->level) {
447 : head = &pos->list;
448 : break;
449 : }
450 3852 : if (frame->level > pos->level)
451 1540 : continue;
452 2312 : if (frame->level < pos->level) {
453 : head = &pos->list;
454 : break;
455 : }
456 : }
457 179933 : if (!frame->ack_eliciting) {
458 : head = &pos->list;
459 : break;
460 : }
461 22906 : if (!frame->level)
462 : break;
463 : }
464 :
465 563496 : outq->unsent_bytes += frame->bytes;
466 563496 : quic_outq_set_owner_w((int)frame->bytes, sk);
467 563495 : list_add_tail(&frame->list, head);
468 563492 : if (!cork)
469 4590 : quic_outq_transmit(sk);
470 563497 : }
471 :
472 : /* Inserts a frame into transmitted_list in order by level and offset (first packet number used). */
473 6524492 : void quic_outq_transmitted_tail(struct sock *sk, struct quic_frame *frame)
474 : {
475 6524492 : struct list_head *head = &quic_outq(sk)->transmitted_list;
476 6524492 : struct quic_frame *pos;
477 :
478 : /* Insert frame in priority order:
479 : *
480 : * Initial (level == 1) > Handshake (level == 2) > Application (level == 0);
481 : * At same level: first packet number used less > first packet number used greater.
482 : */
483 186516385 : list_for_each_entry(pos, head, list) {
484 179996200 : if (!frame->level) {
485 179990090 : if (pos->level)
486 943 : continue;
487 179989147 : goto offset;
488 : }
489 6110 : if (!pos->level) {
490 : head = &pos->list;
491 : break;
492 : }
493 5307 : if (frame->level > pos->level)
494 1761 : continue;
495 3546 : if (frame->level < pos->level) {
496 : head = &pos->list;
497 : break;
498 : }
499 3383 : offset:
500 179992530 : if (frame->offset < pos->offset) {
501 : head = &pos->list;
502 : break;
503 : }
504 : }
505 6524492 : frame->transmitted = 1; /* Mark the frame as added to the transmitted_list. */
506 6524492 : list_add_tail(&frame->list, head);
507 6523680 : }
508 :
509 : /* Inserts a sent packet into packet_sent_list in order by level. */
510 6484612 : void quic_outq_packet_sent_tail(struct sock *sk, struct quic_packet_sent *sent)
511 : {
512 6484612 : struct list_head *head = &quic_outq(sk)->packet_sent_list;
513 6484612 : struct quic_packet_sent *pos;
514 :
515 : /* Insert sent packet in priority order:
516 : *
517 : * Handshake levels (level > 0) > Appliation level (level == 0).
518 : */
519 6484612 : if (sent->level) {
520 5128 : list_for_each_entry(pos, head, list) {
521 3101 : if (!pos->level) {
522 : head = &pos->list;
523 : break;
524 : }
525 2298 : if (sent->level > pos->level)
526 664 : continue;
527 1634 : if (sent->level < pos->level) {
528 : head = &pos->list;
529 : break;
530 : }
531 : }
532 : }
533 6484612 : list_add_tail(&sent->list, head);
534 6484529 : }
535 :
536 : /* Transmit a probe packet (PING frame with padding) to assist with PLPMTUD. */
537 45 : void quic_outq_transmit_probe(struct sock *sk)
538 : {
539 45 : struct quic_pnspace *space = quic_pnspace(sk, QUIC_CRYPTO_APP);
540 45 : u32 taglen = quic_packet_taglen(quic_packet(sk));
541 45 : struct quic_path_group *paths = quic_paths(sk);
542 45 : struct quic_config *c = quic_config(sk);
543 45 : struct quic_probeinfo info;
544 45 : u32 pathmtu;
545 45 : s64 number;
546 :
547 45 : if (!quic_is_established(sk))
548 1 : return;
549 :
550 44 : if (quic_packet_config(sk, QUIC_CRYPTO_APP, 0))
551 : return;
552 :
553 : /* Set probe packet size and encryption level. */
554 44 : info.size = paths->pl.probe_size;
555 44 : info.level = QUIC_CRYPTO_APP;
556 : /* Save the packet number used for confirming the probe via ACK. */
557 44 : number = space->next_pn;
558 44 : if (!quic_outq_transmit_frame(sk, QUIC_FRAME_PING, &info, 0, false)) {
559 44 : pathmtu = quic_path_pl_send(paths, number);
560 44 : if (pathmtu) /* Pathmtu may drop if probe failure count exceeded the limit. */
561 0 : quic_packet_mss_update(sk, pathmtu + taglen);
562 : }
563 :
564 : /* Restart the PLPMTUD timer for future probes if this one fails. */
565 44 : quic_timer_reset(sk, QUIC_TIMER_PMTU, c->plpmtud_probe_interval);
566 : }
567 :
568 : /* Queue and send a CONNECTION_CLOSE frame to terminate the connection. */
569 49 : void quic_outq_transmit_close(struct sock *sk, u8 type, u32 errcode, u8 level)
570 : {
571 49 : struct quic_outqueue *outq = quic_outq(sk);
572 49 : struct quic_connection_close close = {};
573 :
574 49 : if (!errcode)
575 48 : return;
576 :
577 1 : close.errcode = errcode;
578 1 : close.frame = type;
579 1 : quic_inq_event_recv(sk, QUIC_EVENT_CONNECTION_CLOSE, &close);
580 :
581 1 : outq->close_errcode = errcode;
582 1 : outq->close_frame = type;
583 :
584 1 : quic_outq_transmit_frame(sk, QUIC_FRAME_CONNECTION_CLOSE, &level, 0, false);
585 1 : quic_set_state(sk, QUIC_SS_CLOSED);
586 : }
587 :
588 : /* Send an application-level CONNECTION_CLOSE frame, typically called by close() or shutdown(). */
589 1122 : void quic_outq_transmit_app_close(struct sock *sk)
590 : {
591 1122 : u32 errcode = QUIC_TRANSPORT_ERROR_APPLICATION;
592 1122 : u8 type = QUIC_FRAME_CONNECTION_CLOSE, level;
593 1122 : struct quic_outqueue *outq = quic_outq(sk);
594 :
595 1122 : if (quic_is_established(sk)) {
596 : /* Set close_frame so send is not delayed in quic_outq_delay_check(). */
597 616 : level = QUIC_CRYPTO_APP;
598 616 : type = QUIC_FRAME_CONNECTION_CLOSE_APP;
599 616 : outq->close_frame = type;
600 616 : quic_outq_transmit(sk); /* Flush data before sending close frame. */
601 506 : } else if (quic_is_establishing(sk)) {
602 : /* Connection still in handshake: send close in INITIAL level packets. */
603 0 : level = QUIC_CRYPTO_INITIAL;
604 0 : outq->close_errcode = errcode;
605 : } else { /* Connection is already closed: no action needed. */
606 506 : return;
607 : }
608 616 : quic_outq_transmit_frame(sk, type, &level, 0, false);
609 : }
610 :
611 : /* Processes frames in a sent packet that have been acknowledged (SACKed). */
612 6476148 : static void quic_outq_psent_sack_frames(struct sock *sk, struct quic_packet_sent *sent)
613 : {
614 6476148 : struct quic_frame *frame;
615 6476148 : int acked = 0, i;
616 :
617 : /* Release all frames held in this sent packet. */
618 12990988 : for (i = 0; i < sent->frames; i++) {
619 6514838 : frame = sent->frame_array[i];
620 6514838 : if (list_empty(&frame->list)) {
621 : /* It is already ACKed by another packet: just drop reference held in
622 : * frame_array.
623 : */
624 2822 : quic_frame_put(frame);
625 2822 : continue;
626 : }
627 6512016 : quic_frame_put(frame); /* Drop reference held in frame_array. */
628 :
629 6512018 : acked += frame->bytes;
630 : /* Remove from send/transmitted list and drop reference held by it. */
631 6512018 : quic_frame_ack(sk, frame);
632 : }
633 6476150 : quic_outq_wfree(acked, sk);
634 6476150 : }
635 :
636 : #define QUIC_PMTUD_RAISE_TIMER_FACTOR 30
637 :
638 : /* Confirms the path probe and triggers PLPMTUD state machine. */
639 428530 : static void quic_outq_path_confirm(struct sock *sk, u8 level, s64 largest, s64 smallest)
640 : {
641 428530 : struct quic_path_group *paths = quic_paths(sk);
642 428530 : struct quic_outqueue *outq = quic_outq(sk);
643 428530 : struct quic_config *c = quic_config(sk);
644 428530 : bool raise_timer, complete;
645 428530 : u32 pathmtu;
646 :
647 : /* Reset pto_count unless the client is unsure if the server has validated the client's
648 : * address.
649 : */
650 428530 : if (paths->validated)
651 427380 : outq->pto_count = 0;
652 :
653 : /* Check if this packet number confirms PLPMTUD probe in APP level (0). */
654 428530 : if (level || !quic_path_pl_confirm(paths, largest, smallest))
655 428502 : return;
656 :
657 : /* Get new path MTU and check if raise timer is needed. */
658 28 : pathmtu = quic_path_pl_recv(paths, &raise_timer, &complete);
659 28 : if (pathmtu)
660 1 : quic_packet_mss_update(sk, pathmtu + quic_packet_taglen(quic_packet(sk)));
661 28 : if (!complete) /* If PLPMTUD is not complete, continue sending a probe packet. */
662 27 : quic_outq_transmit_probe(sk);
663 28 : if (raise_timer) /* Reset the probe timer as raise timer if needed. */
664 1 : quic_timer_reset(sk, QUIC_TIMER_PMTU,
665 1 : (u64)c->plpmtud_probe_interval * QUIC_PMTUD_RAISE_TIMER_FACTOR);
666 : }
667 :
668 : /* rfc9002#section-a.7: OnAckReceived()
669 : *
670 : * Process ACK reception for transmitted packets: This function identifies newly acknowledged
671 : * packets in the specified packet number space, updates congestion control and RTT measurements,
672 : * removes acknowledged packets from tracking, and adjusts send window and pacing accordingly.
673 : */
674 428530 : void quic_outq_transmitted_sack(struct sock *sk, u8 level, s64 largest, s64 smallest,
675 : s64 ack_largest, u32 ack_delay)
676 : {
677 428530 : struct quic_pnspace *space = quic_pnspace(sk, level);
678 428530 : struct quic_crypto *crypto = quic_crypto(sk, level);
679 428530 : struct quic_outqueue *outq = quic_outq(sk);
680 428530 : struct quic_cong *cong = quic_cong(sk);
681 428530 : struct quic_packet_sent *sent, *next;
682 428530 : u32 acked = 0;
683 :
684 428530 : quic_outq_path_confirm(sk, level, largest, smallest);
685 428530 : pr_debug("%s: largest: %llu, smallest: %llu\n", __func__, largest, smallest);
686 :
687 : /* Iterate backwards over sent packets to efficiently process newly ACKed packets. */
688 10872519 : list_for_each_entry_safe_reverse(sent, next, &outq->packet_sent_list, list) {
689 10447623 : if (level != sent->level)
690 9257 : continue;
691 10438366 : if (sent->number > largest)
692 3960116 : continue;
693 6478250 : if (sent->number < smallest)
694 : break;
695 :
696 : /* rfc9000#section-13.4.2:
697 : *
698 : * To perform ECN validation for a new path:
699 : *
700 : * The endpoint monitors whether all packets sent with an ECT codepoint are
701 : * eventually deemed lost, indicating that ECN validation has failed.
702 : */
703 6474615 : if (sent->ecn)
704 1595 : quic_set_sk_ecn(sk, INET_ECN_ECT_0);
705 :
706 6474615 : outq->inflight -= sent->frame_len;
707 6474615 : space->inflight -= sent->frame_len;
708 : /* Process the frames contained in the acknowledged packet. */
709 6474615 : quic_outq_psent_sack_frames(sk, sent);
710 :
711 6474617 : if (sent->number == ack_largest) {
712 : /* Update the RTT if the largest acknowledged is newly acked. */
713 369410 : quic_pnspace_set_max_pn_acked_seen(space, sent->number);
714 369410 : quic_cong_rtt_update(cong, sent->sent_time, ack_delay);
715 :
716 : /* These two members are calculated based on cong.pto. */
717 369410 : space->max_time_limit = cong->pto * 2;
718 369410 : crypto->key_update_time = cong->pto * 2;
719 : }
720 : /* Call cong.on_packet_acked() and sync send window. */
721 6474617 : quic_cong_on_packet_acked(cong, sent->sent_time, sent->frame_len, sent->number);
722 6474617 : quic_outq_sync_window(sk, cong->window);
723 :
724 6474617 : acked += sent->frame_len;
725 6474617 : list_del(&sent->list);
726 6474617 : kfree(sent);
727 : }
728 :
729 : /* Call cong.on_ack_recv() where it does pacing rate update. */
730 428531 : quic_cong_on_ack_recv(cong, acked, READ_ONCE(sk->sk_max_pacing_rate));
731 428531 : }
732 :
733 : /* rfc9002#section-a.8: GetLossTimeAndSpace()
734 : *
735 : * Find the earliest loss detection timer among the three packet number spaces: Initial,
736 : * Handshake, and Application. Return the earliest loss time and update the level to indicate
737 : * which packet number space it belongs to.
738 : */
739 6874040 : static u32 quic_outq_get_loss_time(struct sock *sk, u8 *level)
740 : {
741 6874040 : struct quic_pnspace *s;
742 6874040 : u32 time, t;
743 :
744 : /* Start with Initial packet number space loss time. */
745 6874040 : s = quic_pnspace(sk, QUIC_CRYPTO_INITIAL);
746 6874040 : t = s->loss_time;
747 6874040 : time = t;
748 6874040 : *level = QUIC_CRYPTO_INITIAL;
749 :
750 : /* Check Handshake packet number space for an earlier loss time. */
751 6874040 : s = quic_pnspace(sk, QUIC_CRYPTO_HANDSHAKE);
752 6874040 : t = s->loss_time;
753 6874040 : if (t && (!time || time > t)) {
754 0 : time = t;
755 0 : *level = QUIC_CRYPTO_HANDSHAKE;
756 : }
757 :
758 : /* Check Application packet number space for an even earlier loss time. */
759 6874040 : s = quic_pnspace(sk, QUIC_CRYPTO_APP);
760 6874040 : t = s->loss_time;
761 6874040 : if (t && (!time || time > t)) {
762 3934 : time = t;
763 3934 : *level = QUIC_CRYPTO_APP;
764 : }
765 :
766 6874040 : return time;
767 : }
768 :
769 : /* rfc9002#section-a.8: GetPtoTimeAndSpace()
770 : *
771 : * Calculate the earliest Probe Timeout (PTO) expiration time across packet number spaces.
772 : * Returns the time at which the PTO expires and updates the level indicating the packet number
773 : * space associated with the PTO timer.
774 : */
775 6710969 : static u32 quic_outq_get_pto_time(struct sock *sk, u8 *level)
776 : {
777 6710969 : u32 duration, t, time = 0, now = jiffies_to_usecs(jiffies);
778 6710983 : struct quic_outqueue *outq = quic_outq(sk);
779 6710983 : struct quic_pnspace *s;
780 :
781 : /* PTO duration scaled by (2 ^ pto_count). */
782 6710983 : duration = quic_cong(sk)->pto * BIT(outq->pto_count);
783 :
784 6710983 : if (!outq->inflight) {
785 : /* If nothing is inflight, PTO is scheduled for the next expected handshake or
786 : * initial packet.
787 : */
788 1207 : *level = QUIC_CRYPTO_INITIAL;
789 1207 : if (quic_crypto(sk, QUIC_CRYPTO_HANDSHAKE)->send_ready)
790 634 : *level = QUIC_CRYPTO_HANDSHAKE;
791 1207 : return now + duration;
792 : }
793 :
794 : /* Check Initial packet space PTO expiration time. */
795 6709776 : s = quic_pnspace(sk, QUIC_CRYPTO_INITIAL);
796 6709776 : if (s->inflight) {
797 3241 : t = s->last_sent_time + duration;
798 3241 : time = t;
799 3241 : *level = QUIC_CRYPTO_INITIAL;
800 : }
801 :
802 : /* Check Handshake packet space PTO expiration time and choose earliest. */
803 6709776 : s = quic_pnspace(sk, QUIC_CRYPTO_HANDSHAKE);
804 6709776 : if (s->inflight) {
805 3477 : t = s->last_sent_time + duration;
806 3477 : if (!time || time > t) {
807 2712 : time = t;
808 2712 : *level = QUIC_CRYPTO_HANDSHAKE;
809 : }
810 : }
811 :
812 6709776 : if (time)
813 : return time;
814 :
815 : /* Check Application packet space PTO expiration time. */
816 6704237 : s = quic_pnspace(sk, QUIC_CRYPTO_APP);
817 6704237 : if (s->inflight) {
818 6704223 : duration += (outq->max_ack_delay * BIT(outq->pto_count));
819 6704223 : t = s->last_sent_time + duration;
820 6704223 : if (!time || time > t) {
821 6704223 : time = t;
822 6704223 : *level = QUIC_CRYPTO_APP;
823 : }
824 : }
825 :
826 : return time;
827 : }
828 :
829 : /* rfc9002#section-a.8: SetLossDetectionTimer()
830 : *
831 : * Update the loss detection timer for the socket based on the earliest loss time or PTO. If no
832 : * loss time is found, and no inflight packets exist but connection is established or path is
833 : * blocked due to anti-amplification limit, stop the loss timer. Otherwise, set it to the
834 : * earliest PTO time.
835 : */
836 6871405 : void quic_outq_update_loss_timer(struct sock *sk)
837 : {
838 6871405 : struct quic_path_group *paths = quic_paths(sk);
839 6871405 : struct quic_outqueue *outq = quic_outq(sk);
840 6871405 : u32 time, now = jiffies_to_usecs(jiffies);
841 6871414 : u8 level;
842 :
843 6871414 : time = quic_outq_get_loss_time(sk, &level);
844 6871415 : if (time)
845 3690 : goto out;
846 :
847 6867725 : if ((!outq->inflight && quic_is_established(sk)) || paths->blocked)
848 159136 : return quic_timer_stop(sk, QUIC_TIMER_LOSS);
849 :
850 6708589 : time = quic_outq_get_pto_time(sk, &level);
851 :
852 6712268 : out:
853 6712268 : time = (time > now) ? (time - now) : 1;
854 6712268 : quic_timer_reset(sk, QUIC_TIMER_LOSS, time);
855 : }
856 :
857 : /* Syncs the congestion window with the socket send buffer size. Called after congestion
858 : * control updates the window.
859 : */
860 6486226 : void quic_outq_sync_window(struct sock *sk, u32 window)
861 : {
862 6486226 : struct quic_outqueue *outq = quic_outq(sk);
863 :
864 6486226 : if (outq->window == window)
865 : return;
866 1361050 : outq->window = window;
867 :
868 1361050 : if (sk->sk_userlocks & SOCK_SNDBUF_LOCK)
869 : return;
870 : /* Dynamically adjust sk_sndbuf based on the congestion window. */
871 1361050 : if (sk->sk_sndbuf > (int)window * 2)
872 1116 : if (sk_stream_wspace(sk) > 0)
873 784 : sk->sk_write_space(sk); /* Wake up processes blocked on sending. */
874 1361050 : sk->sk_sndbuf = (int)window * 2;
875 : }
876 :
877 : /* Put the timeout frame back to the corresponding outqueue for transmitting. */
878 8817 : static void quic_outq_retransmit_frame(struct sock *sk, struct quic_frame *frame)
879 : {
880 8817 : struct quic_outqueue *outq = quic_outq(sk);
881 8817 : struct quic_frame *pos;
882 8817 : struct list_head *head;
883 :
884 8817 : head = &outq->control_list;
885 8817 : if (quic_frame_stream(frame->type)) {
886 7617 : head = &outq->stream_list;
887 :
888 7617 : outq->stream_list_len += frame->len;
889 : }
890 :
891 : /* Insert frame in priority order:
892 : *
893 : * Initial (level == 1) > Handshake (level == 2) > Application (level == 0);
894 : * At same level: first packet number used less > first packet number used greater >
895 : * first packet number used negative.
896 : */
897 1155257 : list_for_each_entry(pos, head, list) {
898 1151223 : if (!frame->level) {
899 1151205 : if (pos->level)
900 0 : continue;
901 1151205 : goto offset;
902 : }
903 18 : if (!pos->level) {
904 : head = &pos->list;
905 : break;
906 : }
907 18 : if (frame->level > pos->level)
908 0 : continue;
909 18 : if (frame->level < pos->level) {
910 : head = &pos->list;
911 : break;
912 : }
913 18 : offset:
914 1151223 : if (pos->offset < 0 || frame->offset < pos->offset) {
915 : head = &pos->list;
916 : break;
917 : }
918 : }
919 8817 : list_add_tail(&frame->list, head);
920 8817 : QUIC_INC_STATS(sock_net(sk), QUIC_MIB_FRM_RETRANS);
921 8817 : }
922 :
923 : /* Retransmits retransmittable frames from a sent packet. Called when a packet is declared lost. */
924 8693 : static void quic_outq_psent_retransmit_frames(struct sock *sk, struct quic_packet_sent *sent)
925 : {
926 8693 : struct quic_frame *frame;
927 8693 : int bytes = 0, i;
928 :
929 19429 : for (i = 0; i < sent->frames; i++) {
930 10736 : frame = sent->frame_array[i];
931 10736 : if (list_empty(&frame->list)) { /* It is already ACKed by another packet. */
932 1851 : quic_frame_put(frame);
933 1851 : continue;
934 : }
935 8885 : quic_frame_put(frame);
936 :
937 8885 : if (!frame->transmitted)
938 68 : continue; /* It is already in queue for transmitting. */
939 :
940 8817 : list_del_init(&frame->list);
941 : /* Don't retransmit DATAGRAM or PING frames. */
942 8817 : if (quic_frame_dgram(frame->type) || quic_frame_ping(frame->type)) {
943 0 : bytes += frame->bytes;
944 0 : quic_frame_put(frame);
945 0 : continue;
946 : }
947 : /* Clear transmitted bit and put it in queue for transmitting. */
948 8817 : frame->transmitted = 0;
949 8817 : quic_outq_retransmit_frame(sk, frame);
950 : }
951 8693 : quic_outq_wfree(bytes, sk);
952 8693 : }
953 :
954 : /* rfc9002#section-a.10: DetectAndRemoveLostPackets()
955 : *
956 : * Identify and mark packets as lost in the specified packet number space. This function scans
957 : * sent packets and moves those considered lost back to the send queue. It updates loss time,
958 : * congestion control state, inflight bytes, and the send window accordingly.
959 : */
960 384390 : void quic_outq_retransmit_mark(struct sock *sk, u8 level, u8 immediate)
961 : {
962 384390 : struct quic_pnspace *space = quic_pnspace(sk, level);
963 384390 : struct quic_outqueue *outq = quic_outq(sk);
964 384390 : struct quic_cong *cong = quic_cong(sk);
965 384390 : struct quic_packet_sent *sent, *next;
966 :
967 384390 : space->loss_time = 0;
968 384390 : cong->time = jiffies_to_usecs(jiffies);
969 :
970 393603 : list_for_each_entry_safe(sent, next, &outq->packet_sent_list, list) {
971 233276 : if (level && !sent->level)
972 : break;
973 232959 : if (level != sent->level)
974 520 : continue;
975 :
976 : /* rfc9002#section-6.1:
977 : *
978 : * A packet is declared lost if it meets all of the following conditions:
979 : *
980 : * - The packet is unacknowledged, and was sent prior to an acknowledged
981 : * packet.
982 : * - The packet was sent kPacketThreshold packets before an acknowledged
983 : * packet, or it was sent long enough (loss_delay) in the past.
984 : */
985 232439 : if (!immediate && sent->number > space->max_pn_acked_seen)
986 : break;
987 :
988 9653 : if (!immediate && sent->sent_time + cong->loss_delay > cong->time &&
989 2853 : sent->number + QUIC_KPACKET_THRESHOLD > space->max_pn_acked_seen) {
990 960 : if (!space->loss_time ||
991 : space->loss_time > sent->sent_time + cong->loss_delay)
992 960 : space->loss_time = sent->sent_time + cong->loss_delay;
993 : break;
994 : }
995 :
996 8693 : outq->inflight -= sent->frame_len;
997 8693 : space->inflight -= sent->frame_len;
998 : /* Move frames from the lost packet back to the send queue. */
999 8693 : quic_outq_psent_retransmit_frames(sk, sent);
1000 :
1001 : /* Call cong.on_packet_lost() and sync send window. */
1002 8693 : quic_cong_on_packet_lost(cong, space->max_pn_acked_time,
1003 8693 : sent->frame_len, sent->number);
1004 8693 : quic_outq_sync_window(sk, cong->window);
1005 :
1006 8693 : list_del(&sent->list);
1007 8693 : kfree(sent);
1008 : }
1009 384390 : }
1010 :
1011 : /* Removes each frame from the list and queues it for retransmission. Called when packet
1012 : * construction fails using frames in the packet list.
1013 : */
1014 0 : void quic_outq_retransmit_list(struct sock *sk, struct list_head *head)
1015 : {
1016 0 : struct quic_frame *frame, *next;
1017 :
1018 : /* Clear transmitted bit and put them in queue for transmitting. */
1019 0 : list_for_each_entry_safe(frame, next, head, list) {
1020 0 : list_del_init(&frame->list);
1021 0 : frame->transmitted = 0;
1022 0 : quic_outq_retransmit_frame(sk, frame);
1023 : }
1024 0 : }
1025 :
1026 : #define QUIC_MAX_PTO_COUNT 8
1027 :
1028 : /* rfc9002#section-a.9: OnLossDetectionTimeout()
1029 : *
1030 : * Handle Probe Timeout (PTO) expiration: This function is invoked when the loss detection timer
1031 : * expires. It attempts to retransmit frames contained in the lost packets if any are detected.
1032 : * Otherwise, it sends probe packets to elicit acknowledgments and maintain connection liveness.
1033 : * It also manages the PTO count and resets the loss timer.
1034 : */
1035 2632 : void quic_outq_transmit_pto(struct sock *sk)
1036 : {
1037 2632 : struct quic_outqueue *outq = quic_outq(sk);
1038 2632 : struct quic_probeinfo info = {};
1039 2632 : u32 time;
1040 2632 : u8 level;
1041 :
1042 : /* Check if there is a loss time set and retransmit lost frames if so. */
1043 2632 : time = quic_outq_get_loss_time(sk, &level);
1044 2631 : if (time) {
1045 : /* Move frames from lost packets back to the send queue, update the loss
1046 : * detection timer, and retransmit the frames.
1047 : */
1048 244 : quic_outq_retransmit_mark(sk, level, 0);
1049 244 : quic_outq_update_loss_timer(sk);
1050 244 : quic_outq_transmit(sk);
1051 244 : return;
1052 : }
1053 :
1054 : /* No loss detected, get PTO time and associated packet number space. */
1055 2387 : quic_outq_get_pto_time(sk, &level);
1056 :
1057 : /* Attempt to send one ACK-eliciting probe packets for PTO. */
1058 2386 : if (quic_outq_transmit_single(sk, level))
1059 2231 : goto out;
1060 :
1061 157 : if (quic_packet_config(sk, level, 0))
1062 0 : goto out;
1063 :
1064 : /* If still no packet can be sent, send a PING frame to elicit ACK. */
1065 157 : if (level) {
1066 149 : info.level = level;
1067 149 : info.size = QUIC_MIN_UDP_PAYLOAD;
1068 : }
1069 157 : quic_outq_transmit_frame(sk, QUIC_FRAME_PING, &info, 0, false);
1070 :
1071 2388 : out:
1072 2388 : if (outq->pto_count < QUIC_MAX_PTO_COUNT)
1073 2309 : outq->pto_count++; /* pto_count is used in quic_outq_get_pto_time(). */
1074 2388 : quic_outq_update_loss_timer(sk);
1075 : }
1076 :
1077 : /* Initiate probing of an alternative QUIC path to support path migration. */
1078 99 : int quic_outq_probe_path_alt(struct sock *sk, u8 cork)
1079 : {
1080 99 : struct quic_conn_id_set *id_set = quic_dest(sk);
1081 99 : struct quic_path_group *paths = quic_paths(sk);
1082 99 : u64 number;
1083 :
1084 : /* Try to select an alternate connection ID for the new path. */
1085 99 : if (!quic_conn_id_select_alt(id_set, false)) {
1086 : /* If a probe is already pending, we cannot proceed. */
1087 0 : if (quic_path_alt_state(paths, QUIC_PATH_ALT_PENDING))
1088 : return -EINVAL;
1089 :
1090 : /* No alternate ID available; retire the old connection ID and request a new
1091 : * connection ID to prepare for migration.
1092 : */
1093 0 : number = quic_conn_id_first_number(id_set);
1094 0 : if (quic_outq_transmit_frame(sk, QUIC_FRAME_RETIRE_CONNECTION_ID, &number, 0, cork))
1095 : return -ENOMEM;
1096 :
1097 : /* Mark path migration as pending. */
1098 0 : quic_path_set_alt_state(paths, QUIC_PATH_ALT_PENDING);
1099 0 : return 0;
1100 : }
1101 :
1102 : /* Alternate connection ID selected; start active probing. */
1103 99 : quic_path_set_alt_state(paths, QUIC_PATH_ALT_PROBING);
1104 99 : quic_set_sk_ecn(sk, 0); /* Clear ECN counters to avoid mixing signals across paths. */
1105 : /* Send PATH_CHALLENGE frame on the new path and reset path timer. */
1106 99 : quic_outq_transmit_frame(sk, QUIC_FRAME_PATH_CHALLENGE, NULL, 1, cork);
1107 99 : quic_timer_reset_path(sk);
1108 99 : return 0;
1109 : }
1110 :
1111 : /* Updates the path ID for all frames in control and transmitted lists. Called after
1112 : * connection migration is completed.
1113 : */
1114 97 : void quic_outq_update_path(struct sock *sk, u8 path)
1115 : {
1116 97 : struct quic_outqueue *outq = quic_outq(sk);
1117 97 : struct quic_frame *pos;
1118 :
1119 138 : list_for_each_entry(pos, &outq->control_list, list)
1120 41 : pos->path = path;
1121 :
1122 795 : list_for_each_entry(pos, &outq->transmitted_list, list)
1123 698 : pos->path = path;
1124 97 : }
1125 :
1126 : /* Removes all frames associated with the given stream from both transmitted and stream lists.
1127 : * Called when resetting a stream.
1128 : */
1129 24 : void quic_outq_stream_list_purge(struct sock *sk, struct quic_stream *stream)
1130 : {
1131 24 : struct quic_outqueue *outq = quic_outq(sk);
1132 24 : struct quic_frame *frame, *next;
1133 24 : int bytes = 0;
1134 :
1135 44 : list_for_each_entry_safe(frame, next, &outq->transmitted_list, list) {
1136 20 : if (frame->stream != stream)
1137 0 : continue;
1138 :
1139 20 : bytes += frame->bytes;
1140 20 : list_del_init(&frame->list);
1141 20 : quic_frame_put(frame);
1142 : }
1143 :
1144 24 : list_for_each_entry_safe(frame, next, &outq->stream_list, list) {
1145 0 : if (frame->stream != stream)
1146 0 : continue;
1147 :
1148 0 : outq->stream_list_len -= frame->len;
1149 0 : bytes += frame->bytes;
1150 0 : list_del_init(&frame->list);
1151 0 : quic_frame_put(frame);
1152 : }
1153 24 : quic_outq_wfree(bytes, sk);
1154 24 : }
1155 :
1156 : /* Create and queue a QUIC control frame for transmission.
1157 : *
1158 : * This function creates a new quic_frame with the given type and data, sets the path for
1159 : * the frame, and appends it to the control frame queue.
1160 : */
1161 559979 : int quic_outq_transmit_frame(struct sock *sk, u8 type, void *data, u8 path, u8 cork)
1162 : {
1163 559979 : struct quic_frame *frame;
1164 :
1165 559979 : frame = quic_frame_create(sk, type, data);
1166 559994 : if (!frame)
1167 : return -ENOMEM;
1168 :
1169 559994 : frame->path = path;
1170 559994 : quic_outq_ctrl_tail(sk, frame, cork);
1171 559994 : return 0;
1172 : }
1173 :
1174 : /* Send NEW_CONNECTION_ID frames.
1175 : *
1176 : * This function sends multiple NEW_CONNECTION_ID frames for any connection IDs with
1177 : * sequence numbers between (last known + 1) and (max_count + prior - 1).
1178 : */
1179 1141 : int quic_outq_transmit_new_conn_id(struct sock *sk, u64 prior, u8 path, u8 cork)
1180 : {
1181 1141 : struct quic_conn_id_set *id_set = quic_source(sk);
1182 1141 : u32 max, seqno;
1183 :
1184 : /* Compute the maximum sequence number to send. */
1185 1141 : max = id_set->max_count + prior - 1;
1186 6984 : for (seqno = quic_conn_id_last_number(id_set) + 1; seqno <= max; seqno++) {
1187 5843 : if (quic_outq_transmit_frame(sk, QUIC_FRAME_NEW_CONNECTION_ID, &prior,
1188 : path, true))
1189 : return -ENOMEM;
1190 : }
1191 1141 : if (!cork)
1192 1013 : quic_outq_transmit(sk);
1193 : return 0;
1194 : }
1195 :
1196 : /* Send RETIRE_CONNECTION_ID frames.
1197 : *
1198 : * This function queues RETIRE_CONNECTION_ID frames for all sequence numbers from the first
1199 : * known ID up to the specified prior sequence number.
1200 : */
1201 97 : int quic_outq_transmit_retire_conn_id(struct sock *sk, u64 prior, u8 path, u8 cork)
1202 : {
1203 97 : struct quic_conn_id_set *id_set = quic_dest(sk);
1204 97 : u64 seqno;
1205 :
1206 254 : for (seqno = quic_conn_id_first_number(id_set); seqno < prior; seqno++) {
1207 157 : if (quic_outq_transmit_frame(sk, QUIC_FRAME_RETIRE_CONNECTION_ID, &seqno,
1208 : path, cork))
1209 : return -ENOMEM;
1210 : }
1211 97 : if (!cork)
1212 20 : quic_outq_transmit(sk);
1213 : return 0;
1214 : }
1215 :
1216 : /* Workqueue handler to transmit encrypted QUIC packets. */
1217 0 : static void quic_outq_encrypted_work(struct work_struct *work)
1218 : {
1219 0 : struct quic_sock *qs = container_of(work, struct quic_sock, outq.work);
1220 0 : struct sock *sk = &qs->inet.sk;
1221 0 : struct sk_buff_head *head;
1222 0 : struct quic_skb_cb *cb;
1223 0 : struct sk_buff *skb;
1224 :
1225 0 : lock_sock(sk);
1226 0 : head = &sk->sk_write_queue;
1227 0 : if (sock_flag(sk, SOCK_DEAD)) { /* If the socket is already dead, drop all pending skbs. */
1228 0 : skb_queue_purge(head);
1229 0 : goto out;
1230 : }
1231 :
1232 0 : skb = skb_dequeue(head);
1233 0 : while (skb) {
1234 0 : cb = QUIC_SKB_CB(skb);
1235 0 : if (quic_packet_config(sk, cb->level, cb->path)) {
1236 0 : kfree_skb(skb);
1237 0 : skb = skb_dequeue(head);
1238 0 : continue;
1239 : }
1240 0 : cb->resume = 1; /* Mark this skb encrypted already before sending. */
1241 0 : quic_packet_xmit(sk, skb);
1242 0 : skb = skb_dequeue(head);
1243 : }
1244 0 : quic_packet_flush(sk);
1245 0 : out:
1246 0 : release_sock(sk);
1247 0 : sock_put(sk); /* Drop the hold from quic_outq_encrypted_tail(). */
1248 0 : }
1249 :
1250 : /* Queue an encrypted SKB and schedule transmission.
1251 : *
1252 : * This function queues a fully encrypted skb for asynchronous transmission and schedules
1253 : * the workqueue to process it.
1254 : */
1255 0 : void quic_outq_encrypted_tail(struct sock *sk, struct sk_buff *skb)
1256 : {
1257 0 : struct quic_outqueue *outq = quic_outq(sk);
1258 :
1259 0 : sock_hold(sk);
1260 : /* Add skb to write queue, and send it later in quic_outq_encrypted_work(). */
1261 0 : skb_queue_tail(&sk->sk_write_queue, skb);
1262 :
1263 : /* Schedule work to process queued encrypted packets. If work was already pending,
1264 : * drop the extra hold.
1265 : */
1266 0 : if (!schedule_work(&outq->work))
1267 0 : sock_put(sk);
1268 0 : }
1269 :
1270 : /* Configure outqueue from transport parameters. */
1271 3172 : void quic_outq_set_param(struct sock *sk, struct quic_transport_param *p)
1272 : {
1273 3172 : struct quic_packet *packet = quic_packet(sk);
1274 3172 : struct quic_outqueue *outq = quic_outq(sk);
1275 3172 : struct quic_cong *cong = quic_cong(sk);
1276 3172 : u32 pmtu;
1277 :
1278 3172 : if (!p->remote)
1279 : return;
1280 :
1281 1004 : outq->disable_compatible_version = p->disable_compatible_version;
1282 1004 : outq->disable_1rtt_encryption = p->disable_1rtt_encryption;
1283 1004 : outq->max_datagram_frame_size = p->max_datagram_frame_size;
1284 1004 : outq->max_udp_payload_size = p->max_udp_payload_size;
1285 1004 : outq->ack_delay_exponent = p->ack_delay_exponent;
1286 1004 : outq->max_idle_timeout = p->max_idle_timeout;
1287 1004 : outq->grease_quic_bit = p->grease_quic_bit;
1288 1004 : outq->stateless_reset = p->stateless_reset;
1289 1004 : outq->max_ack_delay = p->max_ack_delay;
1290 1004 : outq->max_data = p->max_data;
1291 :
1292 1004 : outq->max_bytes = outq->max_data;
1293 1004 : cong->max_window = min_t(u64, outq->max_data, S32_MAX / 2);
1294 1004 : cong->max_ack_delay = outq->max_ack_delay;
1295 :
1296 1004 : if (quic_packet_route(sk) < 0)
1297 : return;
1298 1004 : pmtu = min_t(u32, dst_mtu(__sk_dst_get(sk)), QUIC_PATH_MAX_PMTU);
1299 1004 : quic_packet_mss_update(sk, pmtu - packet->hlen);
1300 : }
1301 :
1302 : /* Populate transport parameters from outqueue. */
1303 2079 : void quic_outq_get_param(struct sock *sk, struct quic_transport_param *p)
1304 : {
1305 2079 : struct quic_outqueue *outq = quic_outq(sk);
1306 :
1307 2079 : if (!p->remote)
1308 : return;
1309 :
1310 24 : p->disable_compatible_version = outq->disable_compatible_version;
1311 24 : p->disable_1rtt_encryption = outq->disable_1rtt_encryption;
1312 24 : p->max_datagram_frame_size = outq->max_datagram_frame_size;
1313 24 : p->max_udp_payload_size = outq->max_udp_payload_size;
1314 24 : p->ack_delay_exponent = outq->ack_delay_exponent;
1315 24 : p->max_idle_timeout = outq->max_idle_timeout;
1316 24 : p->grease_quic_bit = outq->grease_quic_bit;
1317 24 : p->stateless_reset = outq->stateless_reset;
1318 24 : p->max_ack_delay = outq->max_ack_delay;
1319 24 : p->max_data = outq->max_data;
1320 : }
1321 :
1322 1119 : void quic_outq_init(struct sock *sk)
1323 : {
1324 1119 : struct quic_outqueue *outq = quic_outq(sk);
1325 :
1326 1119 : INIT_LIST_HEAD(&outq->stream_list);
1327 1119 : INIT_LIST_HEAD(&outq->control_list);
1328 1119 : INIT_LIST_HEAD(&outq->datagram_list);
1329 1119 : INIT_LIST_HEAD(&outq->transmitted_list);
1330 1119 : INIT_LIST_HEAD(&outq->packet_sent_list);
1331 1119 : INIT_WORK(&outq->work, quic_outq_encrypted_work);
1332 1119 : }
1333 :
1334 1118 : static void quic_outq_psent_list_purge(struct sock *sk, struct list_head *head)
1335 : {
1336 1118 : struct quic_packet_sent *sent, *next;
1337 :
1338 2651 : list_for_each_entry_safe(sent, next, head, list) {
1339 1533 : quic_outq_psent_sack_frames(sk, sent);
1340 1533 : list_del(&sent->list);
1341 1533 : kfree(sent);
1342 : }
1343 1118 : }
1344 :
1345 4472 : static void quic_outq_list_purge(struct sock *sk, struct list_head *head)
1346 : {
1347 4472 : struct quic_frame *frame, *next;
1348 4472 : int bytes = 0;
1349 :
1350 4527 : list_for_each_entry_safe(frame, next, head, list) {
1351 55 : bytes += frame->bytes;
1352 55 : list_del_init(&frame->list);
1353 55 : quic_frame_put(frame);
1354 : }
1355 4472 : quic_outq_wfree(bytes, sk);
1356 4472 : }
1357 :
1358 1118 : void quic_outq_free(struct sock *sk)
1359 : {
1360 1118 : struct quic_outqueue *outq = quic_outq(sk);
1361 :
1362 1118 : quic_outq_psent_list_purge(sk, &outq->packet_sent_list);
1363 1118 : quic_outq_list_purge(sk, &outq->transmitted_list);
1364 1118 : quic_outq_list_purge(sk, &outq->datagram_list);
1365 1118 : quic_outq_list_purge(sk, &outq->control_list);
1366 1118 : quic_outq_list_purge(sk, &outq->stream_list);
1367 1118 : __skb_queue_purge(&sk->sk_write_queue);
1368 1118 : kfree(outq->close_phrase);
1369 1118 : }
|
