Skip to content

Commit f75514f

Browse files
author
Alexei Starovoitov
committed
Merge branch 'bpf-next/master' into for-next
Signed-off-by: Alexei Starovoitov <[email protected]>
2 parents c667d55 + 1cb0f56 commit f75514f

File tree

18 files changed

+624
-133
lines changed

18 files changed

+624
-133
lines changed

‎Documentation/bpf/bpf_iterators.rst

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -52,14 +52,14 @@ a pointer to this `struct bpf_iter_<type>` as the very first argument.
5252

5353
Additionally:
5454
- Constructor, i.e., `bpf_iter_<type>_new()`, can have arbitrary extra
55-
number of arguments. Return type is not enforced either.
55+
number of arguments. Return type is not enforced either.
5656
- Next method, i.e., `bpf_iter_<type>_next()`, has to return a pointer
57-
type and should have exactly one argument: `struct bpf_iter_<type> *`
58-
(const/volatile/restrict and typedefs are ignored).
57+
type and should have exactly one argument: `struct bpf_iter_<type> *`
58+
(const/volatile/restrict and typedefs are ignored).
5959
- Destructor, i.e., `bpf_iter_<type>_destroy()`, should return void and
60-
should have exactly one argument, similar to the next method.
60+
should have exactly one argument, similar to the next method.
6161
- `struct bpf_iter_<type>` size is enforced to be positive and
62-
a multiple of 8 bytes (to fit stack slots correctly).
62+
a multiple of 8 bytes (to fit stack slots correctly).
6363

6464
Such strictness and consistency allows to build generic helpers abstracting
6565
important, but boilerplate, details to be able to use open-coded iterators

‎Documentation/bpf/kfuncs.rst

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,23 @@ Or::
160160
...
161161
}
162162

163+
2.2.6 __prog Annotation
164+
---------------------------
165+
This annotation is used to indicate that the argument needs to be fixed up to
166+
the bpf_prog_aux of the caller BPF program. Any value passed into this argument
167+
is ignored, and rewritten by the verifier.
168+
169+
An example is given below::
170+
171+
__bpf_kfunc int bpf_wq_set_callback_impl(struct bpf_wq *wq,
172+
int (callback_fn)(void *map, int *key, void *value),
173+
unsigned int flags,
174+
void *aux__prog)
175+
{
176+
struct bpf_prog_aux *aux = aux__prog;
177+
...
178+
}
179+
163180
.. _BPF_kfunc_nodef:
164181

165182
2.3 Using an existing kernel function

‎arch/s390/net/bpf_jit_comp.c

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -605,17 +605,15 @@ static void bpf_jit_prologue(struct bpf_jit *jit, struct bpf_prog *fp,
605605
}
606606
/* Setup stack and backchain */
607607
if (is_first_pass(jit) || (jit->seen & SEEN_STACK)) {
608-
if (is_first_pass(jit) || (jit->seen & SEEN_FUNC))
609-
/* lgr %w1,%r15 (backchain) */
610-
EMIT4(0xb9040000, REG_W1, REG_15);
608+
/* lgr %w1,%r15 (backchain) */
609+
EMIT4(0xb9040000, REG_W1, REG_15);
611610
/* la %bfp,STK_160_UNUSED(%r15) (BPF frame pointer) */
612611
EMIT4_DISP(0x41000000, BPF_REG_FP, REG_15, STK_160_UNUSED);
613612
/* aghi %r15,-STK_OFF */
614613
EMIT4_IMM(0xa70b0000, REG_15, -(STK_OFF + stack_depth));
615-
if (is_first_pass(jit) || (jit->seen & SEEN_FUNC))
616-
/* stg %w1,152(%r15) (backchain) */
617-
EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0,
618-
REG_15, 152);
614+
/* stg %w1,152(%r15) (backchain) */
615+
EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0,
616+
REG_15, 152);
619617
}
620618
}
621619

@@ -2585,9 +2583,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
25852583
if (nr_stack_args > MAX_NR_STACK_ARGS)
25862584
return -ENOTSUPP;
25872585

2588-
/* Return to %r14, since func_addr and %r0 are not available. */
2589-
if ((!func_addr && !(flags & BPF_TRAMP_F_ORIG_STACK)) ||
2590-
(flags & BPF_TRAMP_F_INDIRECT))
2586+
/* Return to %r14 in the struct_ops case. */
2587+
if (flags & BPF_TRAMP_F_INDIRECT)
25912588
flags |= BPF_TRAMP_F_SKIP_FRAME;
25922589

25932590
/*

‎include/linux/bpf.h

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -346,6 +346,12 @@ static inline const char *btf_field_type_name(enum btf_field_type type)
346346
}
347347
}
348348

349+
#if IS_ENABLED(CONFIG_DEBUG_KERNEL)
350+
#define BPF_WARN_ONCE(cond, format...) WARN_ONCE(cond, format)
351+
#else
352+
#define BPF_WARN_ONCE(cond, format...) BUILD_BUG_ON_INVALID(cond)
353+
#endif
354+
349355
static inline u32 btf_field_type_size(enum btf_field_type type)
350356
{
351357
switch (type) {
@@ -1349,6 +1355,20 @@ u32 __bpf_dynptr_size(const struct bpf_dynptr_kern *ptr);
13491355
const void *__bpf_dynptr_data(const struct bpf_dynptr_kern *ptr, u32 len);
13501356
void *__bpf_dynptr_data_rw(const struct bpf_dynptr_kern *ptr, u32 len);
13511357
bool __bpf_dynptr_is_rdonly(const struct bpf_dynptr_kern *ptr);
1358+
int __bpf_dynptr_write(const struct bpf_dynptr_kern *dst, u32 offset,
1359+
void *src, u32 len, u64 flags);
1360+
void *bpf_dynptr_slice_rdwr(const struct bpf_dynptr *p, u32 offset,
1361+
void *buffer__opt, u32 buffer__szk);
1362+
1363+
static inline int bpf_dynptr_check_off_len(const struct bpf_dynptr_kern *ptr, u32 offset, u32 len)
1364+
{
1365+
u32 size = __bpf_dynptr_size(ptr);
1366+
1367+
if (len > size || offset > size - len)
1368+
return -E2BIG;
1369+
1370+
return 0;
1371+
}
13521372

13531373
#ifdef CONFIG_BPF_JIT
13541374
int bpf_trampoline_link_prog(struct bpf_tramp_link *link,

‎include/linux/bpf_verifier.h

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -591,6 +591,7 @@ struct bpf_insn_aux_data {
591591
* bpf_fastcall pattern.
592592
*/
593593
u8 fastcall_spills_num:3;
594+
u8 arg_prog:4;
594595

595596
/* below fields are initialized once */
596597
unsigned int orig_idx; /* original instruction index */
@@ -838,6 +839,17 @@ __printf(3, 4) void verbose_linfo(struct bpf_verifier_env *env,
838839
u32 insn_off,
839840
const char *prefix_fmt, ...);
840841

842+
#define verifier_bug_if(cond, env, fmt, args...) \
843+
({ \
844+
bool __cond = (cond); \
845+
if (unlikely(__cond)) { \
846+
BPF_WARN_ONCE(1, "verifier bug: " fmt "(" #cond ")\n", ##args); \
847+
bpf_log(&env->log, "verifier bug: " fmt "(" #cond ")\n", ##args); \
848+
} \
849+
(__cond); \
850+
})
851+
#define verifier_bug(env, fmt, args...) verifier_bug_if(1, env, fmt, ##args)
852+
841853
static inline struct bpf_func_state *cur_func(struct bpf_verifier_env *env)
842854
{
843855
struct bpf_verifier_state *cur = env->cur_state;

‎kernel/bpf/bpf_struct_ops.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -601,7 +601,7 @@ int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks,
601601
if (model->ret_size > 0)
602602
flags |= BPF_TRAMP_F_RET_FENTRY_RET;
603603

604-
size = arch_bpf_trampoline_size(model, flags, tlinks, NULL);
604+
size = arch_bpf_trampoline_size(model, flags, tlinks, stub_func);
605605
if (size <= 0)
606606
return size ? : -EFAULT;
607607

‎kernel/bpf/btf.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7660,7 +7660,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog)
76607660
return 0;
76617661

76627662
if (!prog->aux->func_info) {
7663-
bpf_log(log, "Verifier bug\n");
7663+
verifier_bug(env, "func_info undefined");
76647664
return -EFAULT;
76657665
}
76667666

@@ -7684,7 +7684,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog)
76847684
tname = btf_name_by_offset(btf, fn_t->name_off);
76857685

76867686
if (prog->aux->func_info_aux[subprog].unreliable) {
7687-
bpf_log(log, "Verifier bug in function %s()\n", tname);
7687+
verifier_bug(env, "unreliable BTF for function %s()", tname);
76887688
return -EFAULT;
76897689
}
76907690
if (prog_type == BPF_PROG_TYPE_EXT)

‎kernel/bpf/helpers.c

Lines changed: 12 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1714,16 +1714,6 @@ void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr)
17141714
memset(ptr, 0, sizeof(*ptr));
17151715
}
17161716

1717-
static int bpf_dynptr_check_off_len(const struct bpf_dynptr_kern *ptr, u32 offset, u32 len)
1718-
{
1719-
u32 size = __bpf_dynptr_size(ptr);
1720-
1721-
if (len > size || offset > size - len)
1722-
return -E2BIG;
1723-
1724-
return 0;
1725-
}
1726-
17271717
BPF_CALL_4(bpf_dynptr_from_mem, void *, data, u32, size, u64, flags, struct bpf_dynptr_kern *, ptr)
17281718
{
17291719
int err;
@@ -1810,8 +1800,8 @@ static const struct bpf_func_proto bpf_dynptr_read_proto = {
18101800
.arg5_type = ARG_ANYTHING,
18111801
};
18121802

1813-
static int __bpf_dynptr_write(const struct bpf_dynptr_kern *dst, u32 offset, void *src,
1814-
u32 len, u64 flags)
1803+
int __bpf_dynptr_write(const struct bpf_dynptr_kern *dst, u32 offset, void *src,
1804+
u32 len, u64 flags)
18151805
{
18161806
enum bpf_dynptr_type type;
18171807
int err;
@@ -3012,9 +3002,9 @@ __bpf_kfunc int bpf_wq_start(struct bpf_wq *wq, unsigned int flags)
30123002
__bpf_kfunc int bpf_wq_set_callback_impl(struct bpf_wq *wq,
30133003
int (callback_fn)(void *map, int *key, void *value),
30143004
unsigned int flags,
3015-
void *aux__ign)
3005+
void *aux__prog)
30163006
{
3017-
struct bpf_prog_aux *aux = (struct bpf_prog_aux *)aux__ign;
3007+
struct bpf_prog_aux *aux = (struct bpf_prog_aux *)aux__prog;
30183008
struct bpf_async_kern *async = (struct bpf_async_kern *)wq;
30193009

30203010
if (flags)
@@ -3388,6 +3378,14 @@ BTF_ID_FLAGS(func, bpf_iter_kmem_cache_next, KF_ITER_NEXT | KF_RET_NULL | KF_SLE
33883378
BTF_ID_FLAGS(func, bpf_iter_kmem_cache_destroy, KF_ITER_DESTROY | KF_SLEEPABLE)
33893379
BTF_ID_FLAGS(func, bpf_local_irq_save)
33903380
BTF_ID_FLAGS(func, bpf_local_irq_restore)
3381+
BTF_ID_FLAGS(func, bpf_probe_read_user_dynptr)
3382+
BTF_ID_FLAGS(func, bpf_probe_read_kernel_dynptr)
3383+
BTF_ID_FLAGS(func, bpf_probe_read_user_str_dynptr)
3384+
BTF_ID_FLAGS(func, bpf_probe_read_kernel_str_dynptr)
3385+
BTF_ID_FLAGS(func, bpf_copy_from_user_dynptr, KF_SLEEPABLE)
3386+
BTF_ID_FLAGS(func, bpf_copy_from_user_str_dynptr, KF_SLEEPABLE)
3387+
BTF_ID_FLAGS(func, bpf_copy_from_user_task_dynptr, KF_SLEEPABLE | KF_TRUSTED_ARGS)
3388+
BTF_ID_FLAGS(func, bpf_copy_from_user_task_str_dynptr, KF_SLEEPABLE | KF_TRUSTED_ARGS)
33913389
BTF_KFUNCS_END(common_btf_ids)
33923390

33933391
static const struct btf_kfunc_id_set common_kfunc_set = {

0 commit comments

Comments
 (0)