Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0-or-later
2 : /* QUIC kernel implementation
3 : * (C) Copyright Red Hat Corp. 2023
4 : *
5 : * This file is part of the QUIC kernel implementation
6 : *
7 : * Initialization/cleanup for QUIC protocol support.
8 : *
9 : * Written or modified by:
10 : * Xin Long <lucien.xin@gmail.com>
11 : */
12 :
13 : #include <net/proto_memory.h>
14 :
15 : #include "socket.h"
16 :
17 : /* Frees socket receive memory resources after read. */
18 577512 : static void quic_inq_rfree(int len, struct sock *sk)
19 : {
20 577512 : if (!len)
21 : return;
22 :
23 555380 : atomic_sub(len, &sk->sk_rmem_alloc);
24 555380 : sk_mem_uncharge(sk, len);
25 : }
26 :
27 : /* Charges socket receive memory for new frame. */
28 6363645 : static void quic_inq_set_owner_r(int len, struct sock *sk)
29 : {
30 6363645 : if (!len)
31 : return;
32 :
33 6363641 : atomic_add(len, &sk->sk_rmem_alloc);
34 6363641 : sk_mem_charge(sk, len);
35 : }
36 :
37 : #define QUIC_INQ_RWND_SHIFT 4
38 :
39 : /* Update receive flow control windows and send MAX_DATA or MAX_STREAM_DATA frames if needed. */
40 551526 : void quic_inq_flow_control(struct sock *sk, struct quic_stream *stream, u32 bytes)
41 : {
42 551526 : struct quic_pnspace *space = quic_pnspace(sk, QUIC_CRYPTO_APP);
43 551526 : struct quic_packet *packet = quic_packet(sk);
44 551526 : struct quic_inqueue *inq = quic_inq(sk);
45 551526 : u32 mss, window;
46 551526 : u8 frame = 0;
47 :
48 551526 : if (!bytes)
49 : return;
50 :
51 551526 : mss = quic_packet_mss(packet);
52 : /* Account for bytes read at both stream and connection levels. */
53 551526 : stream->recv.bytes += bytes;
54 551526 : inq->bytes += bytes;
55 :
56 : /* Check and update connection-level flow control. */
57 551526 : window = inq->max_data;
58 551526 : if (inq->bytes + window - inq->max_bytes >=
59 551526 : max(mss, (window >> QUIC_INQ_RWND_SHIFT))) {
60 : /* Reduce window increment if memory pressure detected. */
61 60622 : if (sk_under_memory_pressure(sk))
62 0 : window >>= 1;
63 : /* Increase advertised max data to received data + window. */
64 60622 : inq->max_bytes = inq->bytes + window;
65 60622 : if (!quic_outq_transmit_frame(sk, QUIC_FRAME_MAX_DATA, inq, 0, true))
66 60622 : frame = 1;
67 : }
68 :
69 : /* Check and update stream-level flow control. */
70 551526 : window = stream->recv.window;
71 551526 : if (stream->recv.state < QUIC_STREAM_RECV_STATE_RECVD &&
72 533826 : stream->recv.bytes + window - stream->recv.max_bytes >=
73 533826 : max(mss, (window >> QUIC_INQ_RWND_SHIFT))) {
74 109073 : if (sk_under_memory_pressure(sk))
75 0 : window >>= 1;
76 109073 : stream->recv.max_bytes = stream->recv.bytes + window;
77 109073 : if (!quic_outq_transmit_frame(sk, QUIC_FRAME_MAX_STREAM_DATA, stream, 0, true))
78 : frame = 1;
79 : }
80 :
81 442453 : if (frame) {
82 154750 : space->need_sack = 1; /* Request an ACK frame to be bundled with it. */
83 154750 : quic_outq_transmit(sk);
84 : }
85 : }
86 :
87 : /* Handle in-order stream frame delivery. */
88 6359791 : static void quic_inq_stream_tail(struct sock *sk, struct quic_stream *stream,
89 : struct quic_frame *frame)
90 : {
91 6359791 : struct quic_inqueue *inq = quic_inq(sk);
92 6359791 : struct quic_stream_update update = {};
93 6359791 : u64 overlap;
94 :
95 : /* Calculate overlap between stream's current recv offset and frame offset. */
96 6359791 : overlap = stream->recv.offset - frame->offset;
97 6359791 : if (overlap) { /* Discard overlapping prefix and adjust memory accounting. */
98 412 : quic_inq_rfree((int)frame->len, sk);
99 412 : frame->data += overlap;
100 412 : frame->len -= overlap;
101 412 : quic_inq_set_owner_r((int)frame->len, sk);
102 412 : frame->offset += overlap;
103 : }
104 6359791 : stream->recv.offset += frame->len; /* Advance the stream's receive offset. */
105 :
106 6359791 : if (frame->stream_fin) {
107 : /* Notify that the stream has been fully received. */
108 17636 : update.id = stream->id;
109 17636 : update.state = QUIC_STREAM_RECV_STATE_RECVD;
110 17636 : update.finalsz = frame->offset + frame->len;
111 17636 : quic_inq_event_recv(sk, QUIC_EVENT_STREAM_UPDATE, &update);
112 :
113 : /* rfc9000#section-3.2:
114 : *
115 : * Once all data for the stream has been received, the receiving part
116 : * enters the "Data Recvd" state.
117 : */
118 17636 : stream->recv.state = update.state;
119 : /* Release stream and update limits to allow opening new streams. */
120 17636 : quic_stream_recv_put(quic_streams(sk), stream, quic_is_serv(sk));
121 : }
122 :
123 6359791 : frame->offset = 0; /* Reset offset as it will be reused as read offset in recvmsg(). */
124 6359791 : if (frame->level) {
125 : /* Stream frame was received at encryption level 0-RTT (early data). Queue it
126 : * into early_list. After the handshake completes and 1-RTT keys are installed,
127 : * these frames will be moved to recv_list for delivery to the application.
128 : */
129 241 : frame->level = 0;
130 241 : list_add_tail(&frame->list, &inq->early_list);
131 241 : return;
132 : }
133 : /* Frame is ready for application delivery: queue in recv_list. */
134 6359550 : list_add_tail(&frame->list, &inq->recv_list);
135 6359550 : sk->sk_data_ready(sk); /* Notify socket that data is available. */
136 : }
137 :
138 : /* Check and optionally charge receive memory for a QUIC socket.
139 : * Equivalent to sk_rmem_schedule().
140 : */
141 6363164 : static bool quic_sk_rmem_schedule(struct sock *sk, int size)
142 : {
143 6363164 : int delta;
144 :
145 6363164 : if (!sk_has_account(sk))
146 : return true;
147 6363164 : delta = size - sk->sk_forward_alloc;
148 8933646 : return delta <= 0 || __sk_mem_schedule(sk, delta, SK_MEM_RECV);
149 : }
150 :
151 : /* Process an incoming QUIC stream frame.
152 : *
153 : * Validates memory limits, flow control limits, and deduplicates before queuing. Inserts frame
154 : * either in-order or out-of-order depending on stream state.
155 : *
156 : * Returns 0 on success, -ENOBUFS if memory/flow limits are hit, or -EINVAL on protocol violation.
157 : */
158 6370599 : int quic_inq_stream_recv(struct sock *sk, struct quic_frame *frame)
159 : {
160 6370599 : u64 offset = frame->offset, off, highest = 0;
161 6370599 : struct quic_stream *stream = frame->stream;
162 6370599 : struct quic_inqueue *inq = quic_inq(sk);
163 6370599 : struct quic_stream_update update = {};
164 6370599 : struct net *net = sock_net(sk);
165 6370599 : s64 stream_id = stream->id;
166 6370599 : struct list_head *head;
167 6370599 : struct quic_frame *pos;
168 :
169 : /* Discard duplicate frames that are fully covered by the current receive offset.
170 : * However, do not discard if this frame carries a FIN and the stream has not yet
171 : * received any FIN, to ensure proper handling of stream termination.
172 : */
173 6370599 : if (stream->recv.offset >= offset + frame->len &&
174 10646 : (stream->recv.state == QUIC_STREAM_RECV_STATE_SIZE_KNOWN ||
175 10597 : !frame->stream_fin)) {
176 10642 : quic_frame_put(frame);
177 10642 : return 0;
178 : }
179 :
180 : /* Check receive buffer size and system limits. */
181 6359957 : quic_inq_set_owner_r((int)frame->len, sk);
182 6359957 : if (sk_rmem_alloc_get(sk) > sk->sk_rcvbuf || !quic_sk_rmem_schedule(sk, frame->len)) {
183 0 : QUIC_INC_STATS(net, QUIC_MIB_FRM_RCVBUFDROP);
184 0 : quic_inq_rfree((int)frame->len, sk);
185 0 : return -ENOBUFS;
186 : }
187 :
188 6359957 : off = offset + frame->len;
189 6359957 : if (off > stream->recv.highest) { /* New data beyond current highest seen. */
190 : /* rfc9000#section-4.1:
191 : *
192 : * A receiver MUST close the connection with an error of type
193 : * FLOW_CONTROL_ERROR if the sender violates the advertised connection or
194 : * stream data limits.
195 : */
196 6348034 : highest = off - stream->recv.highest; /* New data beyond previous highest offset. */
197 6348034 : if (inq->highest + highest > inq->max_bytes ||
198 6348034 : stream->recv.highest + highest > stream->recv.max_bytes) {
199 0 : frame->errcode = QUIC_TRANSPORT_ERROR_FLOW_CONTROL;
200 0 : quic_inq_rfree((int)frame->len, sk);
201 0 : return -ENOBUFS;
202 : }
203 : /* Check for violation of known final size (protocol error). */
204 6348034 : if (stream->recv.finalsz && off > stream->recv.finalsz) {
205 0 : frame->errcode = QUIC_TRANSPORT_ERROR_FINAL_SIZE;
206 0 : quic_inq_rfree((int)frame->len, sk);
207 0 : return -EINVAL;
208 : }
209 : }
210 6359957 : if (!stream->recv.highest && !frame->stream_fin) {
211 : /* Notify if first data received on stream. Skip FIN frame, as it will trigger
212 : * a "Size Known" state later.
213 : */
214 273 : update.id = stream->id;
215 273 : update.state = QUIC_STREAM_RECV_STATE_RECV;
216 273 : quic_inq_event_recv(sk, QUIC_EVENT_STREAM_UPDATE, &update);
217 : }
218 6359957 : head = &inq->stream_list;
219 6359957 : if (stream->recv.offset < offset) { /* Out-of-order: insert in frame list in order. */
220 788098 : list_for_each_entry(pos, head, list) {
221 761805 : if (pos->stream->id < stream_id)
222 15787 : continue;
223 746018 : if (pos->stream->id > stream_id) {
224 : head = &pos->list;
225 : break;
226 : }
227 745276 : if (pos->offset > offset) {
228 : head = &pos->list;
229 : break;
230 : }
231 744218 : if (pos->offset + pos->len >= offset + frame->len &&
232 84 : (pos->stream_fin || !frame->stream_fin)) {
233 : /* Duplicate or overlapping frame. Keep if it has FIN while
234 : * the other does not.
235 : */
236 84 : quic_inq_rfree((int)frame->len, sk);
237 84 : quic_frame_put(frame);
238 84 : return 0;
239 : }
240 : }
241 28093 : if (frame->stream_fin) {
242 : /* rfc9000#section-4.5:
243 : *
244 : * Once a final size for a stream is known, it cannot change. If a
245 : * RESET_STREAM or STREAM frame is received indicating a change in the
246 : * final size for the stream, an endpoint SHOULD respond with an error
247 : * of type FINAL_SIZE_ERROR.
248 : */
249 8 : if (off < stream->recv.highest ||
250 8 : (stream->recv.finalsz && stream->recv.finalsz != off)) {
251 0 : frame->errcode = QUIC_TRANSPORT_ERROR_FINAL_SIZE;
252 0 : quic_inq_rfree((int)frame->len, sk);
253 0 : return -EINVAL;
254 : }
255 : /* Notify that the stream has known the final size. */
256 8 : update.id = stream->id;
257 8 : update.state = QUIC_STREAM_RECV_STATE_SIZE_KNOWN;
258 8 : update.finalsz = off;
259 8 : quic_inq_event_recv(sk, QUIC_EVENT_STREAM_UPDATE, &update);
260 :
261 : /* rfc9000#section-3.2:
262 : *
263 : * When a STREAM frame with a FIN bit is received, the final size of
264 : * the stream is known; The receiving part of the stream then enters
265 : * the "Size Known" state.
266 : */
267 8 : stream->recv.state = update.state;
268 8 : stream->recv.finalsz = update.finalsz;
269 : }
270 28093 : list_add_tail(&frame->list, head);
271 28093 : stream->recv.frags++;
272 28093 : inq->highest += highest;
273 28093 : stream->recv.highest += highest;
274 28093 : return 0;
275 : }
276 :
277 : /* In-order: directly handled and queued. */
278 6331780 : inq->highest += highest;
279 6331780 : stream->recv.highest += highest;
280 6331780 : quic_inq_stream_tail(sk, stream, frame);
281 6331780 : if (!stream->recv.frags)
282 : return 0;
283 :
284 : /* Check the buffered frames list and merge any frames contiguous with the current
285 : * stream offset to maintain ordered data delivery.
286 : */
287 44015 : list_for_each_entry_safe(frame, pos, head, list) {
288 43463 : if (frame->stream->id < stream_id)
289 5144 : continue;
290 38319 : if (frame->stream->id > stream_id)
291 : break;
292 38306 : if (frame->offset > stream->recv.offset)
293 : break;
294 28093 : list_del(&frame->list);
295 28093 : stream->recv.frags--;
296 28093 : if (stream->recv.offset >= frame->offset + frame->len &&
297 82 : (stream->recv.state == QUIC_STREAM_RECV_STATE_RECVD ||
298 82 : !frame->stream_fin)) {
299 : /* Duplicate frame. Do not discard if it has FIN and no FIN seen yet. */
300 82 : quic_inq_rfree((int)frame->len, sk);
301 82 : quic_frame_put(frame);
302 82 : continue;
303 : }
304 28011 : quic_inq_stream_tail(sk, stream, frame);
305 : }
306 : return 0;
307 : }
308 :
309 : /* Purge all pending reassembly frames for a given stream. Called when resetting a stream. */
310 17660 : void quic_inq_stream_list_purge(struct sock *sk, struct quic_stream *stream)
311 : {
312 17660 : struct list_head *head = &quic_inq(sk)->stream_list;
313 17660 : struct quic_frame *frame, *next;
314 17660 : int bytes = 0;
315 :
316 17770 : list_for_each_entry_safe(frame, next, head, list) {
317 110 : if (frame->stream != stream)
318 110 : continue;
319 0 : list_del(&frame->list);
320 0 : bytes += frame->len;
321 0 : quic_frame_put(frame);
322 : }
323 17660 : quic_inq_rfree(bytes, sk);
324 17660 : }
325 :
326 4472 : static void quic_inq_list_purge(struct sock *sk, struct list_head *head)
327 : {
328 4472 : struct quic_frame *frame, *next;
329 4472 : int bytes = 0;
330 :
331 4472 : list_for_each_entry_safe(frame, next, head, list) {
332 0 : list_del(&frame->list);
333 0 : bytes += frame->len;
334 0 : quic_frame_put(frame);
335 : }
336 4472 : quic_inq_rfree(bytes, sk);
337 4472 : }
338 :
339 : /* Handle in-order crypto (handshake) frame delivery.
340 : *
341 : * Similar to quic_inq_stream_tail(), but with special handling for New Session Ticket Message
342 : * in crypto frame (level == 0). Tickets are saved in quic_ticket() and exposed to userspace
343 : * via getsockopt().
344 : */
345 3195 : static void quic_inq_handshake_tail(struct sock *sk, struct quic_frame *frame)
346 : {
347 3195 : struct quic_crypto *crypto = quic_crypto(sk, frame->level);
348 3195 : struct quic_data *ticket = quic_ticket(sk);
349 3195 : struct quic_inqueue *inq = quic_inq(sk);
350 3195 : u64 overlap, type, length;
351 3195 : struct list_head *head;
352 3195 : struct quic_frame *pos;
353 3195 : u32 len;
354 3195 : u8 *p;
355 :
356 3195 : overlap = crypto->recv_offset - frame->offset;
357 3195 : if (overlap) {
358 1 : quic_inq_rfree((int)frame->len, sk);
359 1 : frame->data += overlap;
360 1 : frame->len -= overlap;
361 1 : quic_inq_set_owner_r((int)frame->len, sk);
362 1 : frame->offset += overlap;
363 : }
364 3195 : crypto->recv_offset += frame->len;
365 :
366 3195 : if (frame->level) {
367 : /* For handshake messages, insert frame before any data/event frames. */
368 2724 : head = &inq->recv_list;
369 3918 : list_for_each_entry(pos, head, list) {
370 1194 : if (!pos->level) {
371 : head = &pos->list;
372 : break;
373 : }
374 : }
375 :
376 2724 : frame->offset = 0;
377 2724 : list_add_tail(&frame->list, head);
378 2724 : sk->sk_data_ready(sk);
379 2724 : return;
380 : }
381 :
382 : /* Special handling for New Session Ticket Message (level == 0). */
383 471 : if (!crypto->ticket_ready && crypto->recv_offset <= QUIC_TICKET_MAX_LEN) {
384 : /* Append received frame data to ticket buffer. */
385 471 : quic_data_append(ticket, frame->data, frame->len);
386 : /* Attempt to parse the TLS message if we have at least the 4-byte header. */
387 471 : if (ticket->len >= 4) {
388 471 : p = ticket->data;
389 471 : len = ticket->len;
390 471 : quic_get_int(&p, &len, &type, 1);
391 471 : quic_get_int(&p, &len, &length, 3);
392 : /* If the full TLS message is available, mark the ticket as ready. */
393 471 : if (ticket->len >= length + 4) {
394 : /* Notify userspace with the full ticket message. Applications
395 : * can receive it via the NEW_SESSION_TICKET event or getsockopt().
396 : */
397 470 : crypto->ticket_ready = 1;
398 470 : quic_inq_event_recv(sk, QUIC_EVENT_NEW_SESSION_TICKET, ticket);
399 : }
400 : }
401 : }
402 471 : quic_inq_rfree((int)frame->len, sk);
403 471 : quic_frame_put(frame); /* Data copied to ticket buffer; release the frame. */
404 : }
405 :
406 : /* Process an incoming QUIC crypto (handshake) frame.
407 : *
408 : * This function behaves similarly to quic_inq_stream_recv(), but operates on different crypto
409 : * levels instead of streams. It handles:
410 : *
411 : * Returns: 0 on success, or -ENOBUFS if buffer limits are exceeded.
412 : */
413 3544 : int quic_inq_handshake_recv(struct sock *sk, struct quic_frame *frame)
414 : {
415 3544 : u64 offset = frame->offset, crypto_offset;
416 3544 : struct quic_inqueue *inq = quic_inq(sk);
417 3544 : struct quic_crypto *crypto;
418 3544 : u8 level = frame->level;
419 3544 : struct list_head *head;
420 3544 : struct quic_frame *pos;
421 :
422 3544 : crypto = quic_crypto(sk, level);
423 3544 : crypto_offset = crypto->recv_offset;
424 3544 : pr_debug("%s: recv_offset: %llu, offset: %llu, level: %u, len: %u\n",
425 : __func__, crypto_offset, offset, level, frame->len);
426 :
427 3544 : if (crypto_offset >= offset + frame->len) {
428 345 : quic_frame_put(frame);
429 345 : return 0;
430 : }
431 :
432 3199 : quic_inq_set_owner_r((int)frame->len, sk);
433 3199 : if (sk_rmem_alloc_get(sk) > sk->sk_rcvbuf || !quic_sk_rmem_schedule(sk, frame->len)) {
434 : /* rfc9000#section-7.5:
435 : *
436 : * If an endpoint's buffer is exceeded during the handshake, it can expand its
437 : * buffer temporarily to complete the handshake. If an endpoint does not expand
438 : * its buffer, it MUST close the connection with a CRYPTO_BUFFER_EXCEEDED error
439 : * code.
440 : */
441 0 : QUIC_INC_STATS(sock_net(sk), QUIC_MIB_FRM_RCVBUFDROP);
442 0 : frame->errcode = QUIC_TRANSPORT_ERROR_CRYPTO_BUF_EXCEEDED;
443 0 : quic_inq_rfree((int)frame->len, sk);
444 0 : return -ENOBUFS;
445 : }
446 :
447 3199 : head = &inq->handshake_list;
448 3199 : if (offset > crypto_offset) {
449 43 : list_for_each_entry(pos, head, list) {
450 13 : if (pos->level < level)
451 0 : continue;
452 13 : if (pos->level > level) {
453 : head = &pos->list;
454 : break;
455 : }
456 13 : if (pos->offset > offset) {
457 : head = &pos->list;
458 : break;
459 : }
460 13 : if (pos->offset + pos->len >= offset + frame->len) {
461 4 : quic_inq_rfree((int)frame->len, sk);
462 4 : quic_frame_put(frame);
463 4 : return 0;
464 : }
465 : }
466 30 : list_add_tail(&frame->list, head);
467 30 : return 0;
468 : }
469 :
470 3165 : quic_inq_handshake_tail(sk, frame);
471 :
472 3195 : list_for_each_entry_safe(frame, pos, head, list) {
473 34 : if (frame->level < level)
474 0 : continue;
475 34 : if (frame->level > level)
476 : break;
477 34 : if (frame->offset > crypto->recv_offset)
478 : break;
479 30 : list_del(&frame->list);
480 30 : if (crypto->recv_offset >= frame->offset + frame->len) {
481 0 : quic_inq_rfree((int)frame->len, sk);
482 0 : quic_frame_put(frame);
483 0 : continue;
484 : }
485 30 : quic_inq_handshake_tail(sk, frame);
486 : }
487 : return 0;
488 : }
489 :
490 : /* Populate transport parameters from inqueue. */
491 2079 : void quic_inq_get_param(struct sock *sk, struct quic_transport_param *p)
492 : {
493 2079 : struct quic_inqueue *inq = quic_inq(sk);
494 :
495 2079 : if (p->remote)
496 : return;
497 :
498 2055 : p->disable_compatible_version = inq->disable_compatible_version;
499 2055 : p->disable_1rtt_encryption = inq->disable_1rtt_encryption;
500 2055 : p->max_datagram_frame_size = inq->max_datagram_frame_size;
501 2055 : p->max_udp_payload_size = inq->max_udp_payload_size;
502 2055 : p->ack_delay_exponent = inq->ack_delay_exponent;
503 2055 : p->max_idle_timeout = inq->max_idle_timeout;
504 2055 : p->grease_quic_bit = inq->grease_quic_bit;
505 2055 : p->stateless_reset = inq->stateless_reset;
506 2055 : p->max_ack_delay = inq->max_ack_delay;
507 2055 : p->max_data = inq->max_data;
508 : }
509 :
510 : /* Configure inqueue from transport parameters. */
511 3172 : void quic_inq_set_param(struct sock *sk, struct quic_transport_param *p)
512 : {
513 3172 : struct quic_packet *packet = quic_packet(sk);
514 3172 : struct quic_inqueue *inq = quic_inq(sk);
515 :
516 3172 : if (p->remote) {
517 1004 : if (p->max_idle_timeout &&
518 1002 : (!inq->max_idle_timeout || p->max_idle_timeout < inq->max_idle_timeout))
519 399 : inq->timeout = p->max_idle_timeout;
520 :
521 1004 : if (inq->disable_1rtt_encryption && p->disable_1rtt_encryption)
522 4 : quic_packet_set_taglen(packet, 0);
523 1004 : return;
524 : }
525 :
526 2168 : inq->disable_compatible_version = p->disable_compatible_version;
527 2168 : inq->disable_1rtt_encryption = p->disable_1rtt_encryption;
528 2168 : inq->max_datagram_frame_size = p->max_datagram_frame_size;
529 2168 : inq->max_udp_payload_size = p->max_udp_payload_size;
530 2168 : inq->ack_delay_exponent = p->ack_delay_exponent;
531 2168 : inq->max_idle_timeout = p->max_idle_timeout;
532 2168 : inq->grease_quic_bit = p->grease_quic_bit;
533 2168 : inq->stateless_reset = p->stateless_reset;
534 2168 : inq->max_ack_delay = p->max_ack_delay;
535 2168 : inq->max_data = p->max_data;
536 :
537 2168 : inq->timeout = inq->max_idle_timeout;
538 2168 : inq->max_bytes = inq->max_data;
539 2168 : sk->sk_rcvbuf = (int)p->max_data * 2;
540 : }
541 :
542 : /* Process an incoming QUIC event and handle it for delivery. */
543 153736 : int quic_inq_event_recv(struct sock *sk, u8 event, void *args)
544 : {
545 153736 : struct list_head *head = &quic_inq(sk)->recv_list;
546 153736 : struct quic_frame *frame, *pos;
547 153736 : u32 args_len = 0;
548 153736 : u8 *p;
549 :
550 153736 : if (!event || event > QUIC_EVENT_MAX)
551 : return -EINVAL;
552 :
553 153736 : if (!(quic_inq(sk)->events & BIT(event)))
554 : return 0; /* Event type not subscribed by user. */
555 :
556 68 : switch (event) { /* Determine size of the argument payload based on event type. */
557 : case QUIC_EVENT_STREAM_UPDATE:
558 : args_len = sizeof(struct quic_stream_update);
559 : break;
560 0 : case QUIC_EVENT_STREAM_MAX_DATA:
561 0 : args_len = sizeof(struct quic_stream_max_data);
562 0 : break;
563 8 : case QUIC_EVENT_STREAM_MAX_STREAM:
564 8 : args_len = sizeof(u64);
565 8 : break;
566 8 : case QUIC_EVENT_CONNECTION_ID:
567 8 : args_len = sizeof(struct quic_connection_id_info);
568 8 : break;
569 4 : case QUIC_EVENT_CONNECTION_CLOSE:
570 4 : args_len = sizeof(struct quic_connection_close);
571 4 : p = ((struct quic_connection_close *)args)->phrase;
572 4 : if (*p)
573 8 : args_len += strlen(p) + 1;
574 : break;
575 12 : case QUIC_EVENT_CONNECTION_MIGRATION:
576 : case QUIC_EVENT_KEY_UPDATE:
577 12 : args_len = sizeof(u8);
578 12 : break;
579 4 : case QUIC_EVENT_NEW_SESSION_TICKET:
580 : case QUIC_EVENT_NEW_TOKEN:
581 4 : args_len = ((struct quic_data *)args)->len;
582 4 : args = ((struct quic_data *)args)->data;
583 4 : break;
584 : default:
585 : return -EINVAL;
586 : }
587 :
588 68 : frame = quic_frame_alloc(1 + args_len, NULL, GFP_ATOMIC);
589 68 : if (!frame) {
590 0 : pr_debug("%s: event: %u, args_len: %u\n", __func__, event, args_len);
591 0 : return -ENOMEM;
592 : }
593 68 : p = quic_put_data(frame->data, &event, 1);
594 68 : quic_put_data(p, args, args_len);
595 68 : frame->event = 1; /* Mark this frame as an event. */
596 68 : frame->offset = 0;
597 :
598 : /* Insert event frame ahead of stream or dgram data. */
599 72 : list_for_each_entry(pos, head, list) {
600 12 : if (!pos->level && !pos->event) {
601 : head = &pos->list;
602 : break;
603 : }
604 : }
605 68 : quic_inq_set_owner_r((int)frame->len, sk);
606 68 : list_add_tail(&frame->list, head);
607 68 : sk->sk_data_ready(sk);
608 68 : return 0;
609 : }
610 :
611 : /* Process an incoming QUIC datagram frame. */
612 8 : int quic_inq_dgram_recv(struct sock *sk, struct quic_frame *frame)
613 : {
614 8 : quic_inq_set_owner_r((int)frame->len, sk);
615 8 : if (sk_rmem_alloc_get(sk) > sk->sk_rcvbuf || !quic_sk_rmem_schedule(sk, frame->len)) {
616 0 : QUIC_INC_STATS(sock_net(sk), QUIC_MIB_FRM_RCVBUFDROP);
617 0 : quic_inq_rfree((int)frame->len, sk);
618 0 : return -ENOBUFS;
619 : }
620 :
621 8 : frame->dgram = 1; /* Mark the frame as a datagram and prepare for delivery. */
622 8 : frame->offset = 0;
623 8 : list_add_tail(&frame->list, &quic_inq(sk)->recv_list);
624 8 : sk->sk_data_ready(sk);
625 8 : return 0;
626 : }
627 :
628 554326 : void quic_inq_data_read(struct sock *sk, u32 bytes)
629 : {
630 554326 : quic_inq_rfree((int)bytes, sk);
631 554326 : }
632 :
633 : /* Workqueue handler to process decrypted QUIC packets. */
634 0 : static void quic_inq_decrypted_work(struct work_struct *work)
635 : {
636 0 : struct quic_sock *qs = container_of(work, struct quic_sock, inq.work);
637 0 : struct sock *sk = &qs->inet.sk;
638 0 : struct sk_buff_head *head;
639 0 : struct sk_buff *skb;
640 :
641 0 : lock_sock(sk);
642 0 : head = &sk->sk_receive_queue;
643 0 : if (sock_flag(sk, SOCK_DEAD)) { /* If the socket is already dead, drop all pending skbs. */
644 0 : skb_queue_purge(head);
645 0 : goto out;
646 : }
647 :
648 0 : skb = skb_dequeue(head);
649 0 : while (skb) {
650 0 : QUIC_SKB_CB(skb)->resume = 1; /* Mark skb decrypted already before processing. */
651 0 : quic_packet_process(sk, skb);
652 0 : skb = skb_dequeue(head);
653 : }
654 0 : out:
655 0 : release_sock(sk);
656 0 : sock_put(sk); /* Drop the hold from quic_inq_decrypted_tail(). */
657 0 : }
658 :
659 : /* Queue an decrypted SKB and schedule processing.
660 : *
661 : * This function queues a fully decrypted skb for asynchronous processing and schedules
662 : * the workqueue to process it.
663 : */
664 0 : void quic_inq_decrypted_tail(struct sock *sk, struct sk_buff *skb)
665 : {
666 0 : struct quic_inqueue *inq = quic_inq(sk);
667 :
668 0 : sock_hold(sk);
669 : /* Add skb to receive queue, and process it later in quic_inq_decrypted_work(). */
670 0 : skb_queue_tail(&sk->sk_receive_queue, skb);
671 :
672 : /* Schedule work to process queued decrypted packets. If work was already pending,
673 : * drop the extra hold.
674 : */
675 0 : if (!schedule_work(&inq->work))
676 0 : sock_put(sk);
677 0 : }
678 :
679 2132 : void quic_inq_backlog_tail(struct sock *sk, struct sk_buff *skb)
680 : {
681 2132 : __skb_queue_tail(&quic_inq(sk)->backlog_list, skb);
682 2132 : }
683 :
684 1119 : void quic_inq_init(struct sock *sk)
685 : {
686 1119 : struct quic_inqueue *inq = quic_inq(sk);
687 :
688 1119 : skb_queue_head_init(&inq->backlog_list);
689 1119 : INIT_LIST_HEAD(&inq->handshake_list);
690 1119 : INIT_LIST_HEAD(&inq->stream_list);
691 1119 : INIT_LIST_HEAD(&inq->early_list);
692 1119 : INIT_LIST_HEAD(&inq->recv_list);
693 1119 : INIT_WORK(&inq->work, quic_inq_decrypted_work);
694 1119 : }
695 :
696 1118 : void quic_inq_free(struct sock *sk)
697 : {
698 1118 : struct quic_inqueue *inq = quic_inq(sk);
699 :
700 1118 : __skb_queue_purge(&sk->sk_receive_queue);
701 1118 : __skb_queue_purge(&inq->backlog_list);
702 1118 : quic_inq_list_purge(sk, &inq->handshake_list);
703 1118 : quic_inq_list_purge(sk, &inq->stream_list);
704 1118 : quic_inq_list_purge(sk, &inq->early_list);
705 1118 : quic_inq_list_purge(sk, &inq->recv_list);
706 1118 : }
|
