1 // SPDX-License-Identifier: GPL-2.0
2
3 #include "vmlinux.h"
4
5 #include <bpf/bpf_helpers.h>
6 #include <bpf/bpf_tracing.h>
7
8 char _license[] SEC("license") = "GPL";
9
10 #define USEC_PER_SEC 1000000UL
11
12 #define min(a, b) ((a) < (b) ? (a) : (b))
13
tcp_sk(const struct sock * sk)14 static inline struct tcp_sock *tcp_sk(const struct sock *sk)
15 {
16 return (struct tcp_sock *)sk;
17 }
18
19 SEC("struct_ops/write_sk_pacing_init")
BPF_PROG(write_sk_pacing_init,struct sock * sk)20 void BPF_PROG(write_sk_pacing_init, struct sock *sk)
21 {
22 #ifdef ENABLE_ATOMICS_TESTS
23 __sync_bool_compare_and_swap(&sk->sk_pacing_status, SK_PACING_NONE,
24 SK_PACING_NEEDED);
25 #else
26 sk->sk_pacing_status = SK_PACING_NEEDED;
27 #endif
28 }
29
30 SEC("struct_ops/write_sk_pacing_cong_control")
BPF_PROG(write_sk_pacing_cong_control,struct sock * sk,const struct rate_sample * rs)31 void BPF_PROG(write_sk_pacing_cong_control, struct sock *sk,
32 const struct rate_sample *rs)
33 {
34 const struct tcp_sock *tp = tcp_sk(sk);
35 unsigned long rate =
36 ((tp->snd_cwnd * tp->mss_cache * USEC_PER_SEC) << 3) /
37 (tp->srtt_us ?: 1U << 3);
38 sk->sk_pacing_rate = min(rate, sk->sk_max_pacing_rate);
39 }
40
41 SEC("struct_ops/write_sk_pacing_ssthresh")
BPF_PROG(write_sk_pacing_ssthresh,struct sock * sk)42 __u32 BPF_PROG(write_sk_pacing_ssthresh, struct sock *sk)
43 {
44 return tcp_sk(sk)->snd_ssthresh;
45 }
46
47 SEC("struct_ops/write_sk_pacing_undo_cwnd")
BPF_PROG(write_sk_pacing_undo_cwnd,struct sock * sk)48 __u32 BPF_PROG(write_sk_pacing_undo_cwnd, struct sock *sk)
49 {
50 return tcp_sk(sk)->snd_cwnd;
51 }
52
53 SEC(".struct_ops")
54 struct tcp_congestion_ops write_sk_pacing = {
55 .init = (void *)write_sk_pacing_init,
56 .cong_control = (void *)write_sk_pacing_cong_control,
57 .ssthresh = (void *)write_sk_pacing_ssthresh,
58 .undo_cwnd = (void *)write_sk_pacing_undo_cwnd,
59 .name = "bpf_w_sk_pacing",
60 };
61
