68 lines
1.6 KiB
C
68 lines
1.6 KiB
C
/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
|
|
/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
|
|
#pragma once
|
|
#include "bpf_arena_common.h"
|
|
|
|
#ifndef __round_mask
|
|
#define __round_mask(x, y) ((__typeof__(x))((y)-1))
|
|
#endif
|
|
#ifndef round_up
|
|
#define round_up(x, y) ((((x)-1) | __round_mask(x, y))+1)
|
|
#endif
|
|
|
|
#ifdef __BPF__
|
|
#define NR_CPUS (sizeof(struct cpumask) * 8)
|
|
|
|
static void __arena * __arena page_frag_cur_page[NR_CPUS];
|
|
static int __arena page_frag_cur_offset[NR_CPUS];
|
|
|
|
/* Simple page_frag allocator */
|
|
static inline void __arena* bpf_alloc(unsigned int size)
|
|
{
|
|
__u64 __arena *obj_cnt;
|
|
__u32 cpu = bpf_get_smp_processor_id();
|
|
void __arena *page = page_frag_cur_page[cpu];
|
|
int __arena *cur_offset = &page_frag_cur_offset[cpu];
|
|
int offset;
|
|
|
|
size = round_up(size, 8);
|
|
if (size >= PAGE_SIZE - 8)
|
|
return NULL;
|
|
if (!page) {
|
|
refill:
|
|
page = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0);
|
|
if (!page)
|
|
return NULL;
|
|
cast_kern(page);
|
|
page_frag_cur_page[cpu] = page;
|
|
*cur_offset = PAGE_SIZE - 8;
|
|
obj_cnt = page + PAGE_SIZE - 8;
|
|
*obj_cnt = 0;
|
|
} else {
|
|
cast_kern(page);
|
|
obj_cnt = page + PAGE_SIZE - 8;
|
|
}
|
|
|
|
offset = *cur_offset - size;
|
|
if (offset < 0)
|
|
goto refill;
|
|
|
|
(*obj_cnt)++;
|
|
*cur_offset = offset;
|
|
return page + offset;
|
|
}
|
|
|
|
static inline void bpf_free(void __arena *addr)
|
|
{
|
|
__u64 __arena *obj_cnt;
|
|
|
|
addr = (void __arena *)(((long)addr) & ~(PAGE_SIZE - 1));
|
|
obj_cnt = addr + PAGE_SIZE - 8;
|
|
if (--(*obj_cnt) == 0)
|
|
bpf_arena_free_pages(&arena, addr, 1);
|
|
}
|
|
#else
|
|
static inline void __arena* bpf_alloc(unsigned int size) { return NULL; }
|
|
static inline void bpf_free(void __arena *addr) {}
|
|
#endif
|