Skip to content

kvspace: switch frontend to headlenpow XValues (kvlang #329) #26

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
carsontung666 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

kvspace: switch frontend to headlenpow XValues (kvlang #329) #26

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 16 additions & 1 deletion CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,19 @@ if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
set(CMAKE_INSTALL_PREFIX "/usr" CACHE PATH "" FORCE)
endif()

add_library(kvspace SHARED src/frontend.c)
add_library(kvspace SHARED src/frontend.c src/codec_abi.c src/xvalue_head.c src/xvalue_meta.c)

include(CTest)
if(BUILD_TESTING)
add_executable(test_xvalue_head tests/test_xvalue_head.c)
target_include_directories(test_xvalue_head PRIVATE src)
target_link_libraries(test_xvalue_head PRIVATE kvspace)
add_test(NAME xvalue_head COMMAND test_xvalue_head)
add_executable(test_codec_abi tests/test_codec_abi.c)
target_include_directories(test_codec_abi PRIVATE src)
target_link_libraries(test_codec_abi PRIVATE kvspace)
add_test(NAME codec_abi COMMAND test_codec_abi)
endif()

# 前端 SONAME：libkvspace.so.1
set_target_properties(kvspace PROPERTIES
Expand All @@ -37,6 +49,9 @@ install(TARGETS kvspace
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}/kvspace
)
install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/src/xvalue_head.h
${CMAKE_CURRENT_SOURCE_DIR}/src/xvalue_meta.h
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/kvspace)

configure_file(kvspace.pc.in ${CMAKE_CURRENT_BINARY_DIR}/kvspace.pc @ONLY)
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/kvspace.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)
Expand Down
145 changes: 42 additions & 103 deletions cli/kvspace.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ struct Head {
langtype: [u8; 256],
langtype_len: i32,
body_offset: i32,
body_cap: u64,
}

fn zero_head() -> Head {
Expand Down Expand Up @@ -46,26 +47,13 @@ extern "C" {
out: *mut *mut u8,
out_len: *mut u32,
) -> c_int;
// 写即构造：key 已存在且 body_len 相等 → 返回原 box body 偏移指针（就地改）；否则新位置写。
fn kvspaceWriteInPlace(
fn kvspaceSetValue(
h: *mut c_void,
key: *const c_char,
resolve: c_int,
body_len: u32,
body: *mut *mut u8,
err: *mut c_char,
err_cap: u32,
) -> c_int;
fn kvspaceWriteNewPlace(
h: *mut c_void,
key: *const c_char,
r#ref: u8,
storetype: u8,
value: *const u8,
value_len: u32,
ro: u8,
vid: u32,
langtype: *const c_char,
body_len: u32,
body: *mut *mut u8,
err: *mut c_char,
err_cap: u32,
) -> c_int;
Expand Down Expand Up @@ -155,6 +143,7 @@ extern "C" {
fn kvspaceNewInt64(v: i64, out: *mut *mut u8, out_len: *mut u32) -> c_int;
fn kvspaceNewFloat64(v: f64, out: *mut *mut u8, out_len: *mut u32) -> c_int;
fn kvspaceDecodeHead(data: *const u8, data_len: u32, out: *mut Head) -> c_int;
fn kvspaceGetHead(h: *mut c_void, key: *const c_char, out: *mut Head) -> c_int;
}

fn cs(s: &str) -> *const c_char {
Expand Down Expand Up @@ -205,6 +194,9 @@ fn decode(data: &[u8]) -> Value {

/// langtype 无 ref 前缀（ref 是独立字段）：只剥前导 [dims]，其余为基 kind。
fn parse_langtype(lt: &str) -> (Vec<i32>, String) {
if lt.contains('·') {
return (Vec::new(), lt.to_string());
}
if let Some(rest) = lt.strip_prefix('[') {
if let Some(end) = rest.find(']') {
let dims: Vec<i32> = rest[..end]
Expand Down Expand Up @@ -307,27 +299,18 @@ fn parse_value(raw: &str) -> Vec<u8> {
Some(i) => (&rest[..i], &rest[i + 1..]),
None => ("", rest),
};
unsafe { kvspaceNewPtr(cs(k), cs(t), &mut out, &mut len) == 0 }
} else if raw.starts_with("map") {
let dims: Vec<i32> = raw
.trim_start_matches("map")
.trim_matches([':', '[', ']'])
.split(',')
.filter(|s| !s.is_empty())
.map(|s| s.parse().unwrap_or(0))
.collect();
if dims.is_empty() {
fatalf("map 需要 dims，如 map[2,3]:");
unsafe { kvspaceNewPtr(cs(t), cs(k), &mut out, &mut len) == 0 }
} else if let Some(langtype) = raw.strip_prefix("map:") {
if !langtype.contains('·') {
fatalf("map requires key·value langtype");
}
let empty = [0u8; 0];
let ndim = dims.len() as i32;
unsafe {
kvspaceTlvEncode(
cs("stringkeymap"),
empty.as_ptr(),
cs(langtype),
std::ptr::null(),
0,
std::ptr::null(),
0,
dims.as_ptr(),
ndim,
&mut out,
&mut len,
) == 0
Expand Down Expand Up @@ -372,20 +355,7 @@ fn parse_value(raw: &str) -> Vec<u8> {
"nil" => {
return vec![];
}
"index" => {
let zero = [0u8; 4];
unsafe {
kvspaceTlvEncode(
cs("index"),
zero.as_ptr(),
4,
std::ptr::null(),
0,
&mut out,
&mut len,
) == 0
}
}
"index" => fatalf("use mkindex for directories"),
_ => {
fatalf(&format!("unknown kind: {:?}", kind));
}
Expand All @@ -401,61 +371,34 @@ fn parse_value(raw: &str) -> Vec<u8> {
}
}

// 写即构造：解 val 的 TLV 头取 (kindexpr, body)，向后端要可写 body 偏移指针后直接写字节——
// key 已存在且 body 尺寸不变 → WriteInPlace（原 box 就地）；否则 WriteNewPlace（分配新 box + 写 head）。
fn set_one(kv: *mut c_void, key: &str, val: &[u8], ro: u8, vid: u32) {
if val.is_empty() {
return; // nil：不写
}
let mut h = zero_head();
unsafe {
kvspaceDecodeHead(val.as_ptr(), val.len() as u32, &mut h);
let key_ptr = cs(key);
let mut err = [0u8; 256];
unsafe { kvspaceDel(kv, &key_ptr, 1, err.as_mut_ptr() as *mut c_char, 256) };
return;
}
let langtype = std::ffi::CString::new(langtype_bytes(&h)).unwrap();
let off = h.body_offset as usize;
let blen = h.body_len.max(0) as usize;
let body = &val[off..off + blen];
let ck = cs(key);
let mut bp: *mut u8 = std::ptr::null_mut();
let mut err = [0u8; 256];
unsafe {
// ro/vid 属 head，就地写不改 head——显式给定时直接走新位置写。
let rc = if ro != 0 || vid != 0 {
1
} else {
kvspaceWriteInPlace(
kv,
ck,
1,
blen as u32,
&mut bp,
err.as_mut_ptr() as *mut c_char,
256,
)
};
if rc != 0 {
let rc2 = kvspaceWriteNewPlace(
kv,
ck,
h.r#ref,
h.storetype,
ro,
vid,
langtype.as_ptr(),
blen as u32,
&mut bp,
err.as_mut_ptr() as *mut c_char,
256,
);
if rc2 != 0 {
let n = err.iter().position(|&b| b == 0).unwrap_or(err.len());
eprintln!("set {}: {}", key, String::from_utf8_lossy(&err[..n]));
std::process::exit(1);
}
}
if !body.is_empty() && !bp.is_null() {
std::ptr::copy_nonoverlapping(body.as_ptr(), bp, body.len());
}
let rc = unsafe {
kvspaceSetValue(
kv,
ck,
val.as_ptr(),
val.len() as u32,
ro,
vid,
err.as_mut_ptr() as *mut c_char,
256,
)
};
if rc != 0 {
let n = err.iter().position(|&b| b == 0).unwrap_or(err.len());
fatalf(&format!(
"set {}: {}",
key,
String::from_utf8_lossy(&err[..n])
));
}
}

Expand Down Expand Up @@ -566,14 +509,10 @@ fn main() {
}
"head" => {
for k in tail {
let v = get(kv, k);
if v.is_empty() {
let mut h = zero_head();
if unsafe { kvspaceGetHead(kv, cs(k), &mut h) } != 0 {
println!("{}\t(nil)", k);
} else {
let mut h = zero_head();
unsafe {
kvspaceDecodeHead(v.as_ptr(), v.len() as u32, &mut h);
}
let lt = String::from_utf8_lossy(langtype_bytes(&h)).into_owned();
let (_, kind) = parse_langtype(&lt);
let dims = &h.dims[..(h.ndim.max(0) as usize).min(8)];
Expand Down
10 changes: 2 additions & 8 deletions include/kvspace/const.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,10 @@
#define KVSPACE_PATH_SEP "/"
#define KVSPACE_DIR_INDEX_SUF "/"
#define KVSPACE_MEMBER_SEP "·"
#define KVSPACE_INDEX_VALUE_SEP "\n"
#define KVSPACE_RUNTIME_MEMBER_SEP "\xE2\x80\xA5"
#define KVSPACE_EXT_INDEX_HEAD "\xE2\x80\xA6"

#define KVSPACE_KIND_NONE "None"
#define KVSPACE_KIND_BYTE "byte"
#define KVSPACE_KIND_BOOL "bool"
#define KVSPACE_KIND_INT8 "int8"
#define KVSPACE_KIND_INT16 "int16"
Expand All @@ -31,8 +30,6 @@
#define KVSPACE_KIND_CHAR_UTF8 "char/utf8"
#define KVSPACE_KIND_CHAR_ASCII "char/ascii"
#define KVSPACE_KIND_MAP "stringkeymap"
#define KVSPACE_KIND_INDEX "index"
#define KVSPACE_KIND_EXT_INDEX "extindex"
#define KVSPACE_KIND_RWIR "rwir"
#define KVSPACE_KIND_RWFUNC "rwfunc"
#define KVSPACE_KIND_DEF_RWIR "def rwir"
Expand All @@ -47,10 +44,9 @@
X(KVSPACE_PATH_SEP) \
X(KVSPACE_DIR_INDEX_SUF) \
X(KVSPACE_MEMBER_SEP) \
X(KVSPACE_INDEX_VALUE_SEP) \
X(KVSPACE_RUNTIME_MEMBER_SEP) \
X(KVSPACE_EXT_INDEX_HEAD) \
X(KVSPACE_KIND_NONE) \
X(KVSPACE_KIND_BYTE) \
X(KVSPACE_KIND_BOOL) \
X(KVSPACE_KIND_INT8) \
X(KVSPACE_KIND_INT16) \
Expand All @@ -66,8 +62,6 @@
X(KVSPACE_KIND_CHAR_UTF8) \
X(KVSPACE_KIND_CHAR_ASCII) \
X(KVSPACE_KIND_MAP) \
X(KVSPACE_KIND_INDEX) \
X(KVSPACE_KIND_EXT_INDEX) \
X(KVSPACE_KIND_RWIR) \
X(KVSPACE_KIND_RWFUNC) \
X(KVSPACE_KIND_DEF_RWIR) \
Expand Down
55 changes: 23 additions & 32 deletions include/kvspace/kvspace.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,9 @@
* shm://... → libkvspace-c.so.1
* 其余（redis/fs/s3）→ libkvspace_durable.so.1
*
* 头格式（byte-identical，两边一致，由前端静态实现 codec）：
* head = [headlen u16 LE][ref u8][storetype u8][ro u8][vid u32 LE][body_len u32 LE]
* [storetype 物理字段（变长，按 storetype）][langtype kindexpr 串（占至 headlen）]
* body = [body_len B raw]
*
* 三正交轴：
* ref 存储位置：0=inline（body=值本体）/1=ptr（body=目标 key）/2=@ext（body=扩展定位符）
* storetype 物理布局（codec 唯一分派）：NONE / ATOM / ARRAYND / index / extindex
* langtype 语义类型真相：完整 kindexpr 串（含 [dims]、map key·value、struct 原型路径、def 族名），
* 恒为 head 最后一段，无独立长度字段（长度 = headlen − 当前偏移），不含 ptr/ext 前缀。
* 物理字段：ARRAYND = ndim u8 + dims[ndim] u32 LE；index/extindex = 成员名矩阵 dims=[len,cap,M]。
* body 起始偏移 = headlen。None：storetype=NONE、langtype=""、body 空。
* Wire: [pow:u8][flags:u8][a:u64le][b:u64le][langtype][padding][body].
* Head length is 1 << pow; flags hold the storage class and pointer bit.
* ro/vid are stored under /.kvspace-meta/, outside the XValue.
*/

#ifndef KVSPACE_H
Expand All @@ -31,31 +22,31 @@
extern "C" {
#endif

/* ref：存储位置维（head 第 2 字节）。只决定 body 语义与是否间接寻址。 */
#define KVSPACE_REF_INLINE 0 /* body = 值本体 raw */
#define KVSPACE_REF_PTR 1 /* body = 目标 key 路径（软链接，单跳同型） */
#define KVSPACE_REF_EXT 2 /* body = 扩展世界定位符（fs 文件 / gpu tensordata） */
/* Decoded location kind. */
#define KVSPACE_REF_INLINE 0
#define KVSPACE_REF_PTR 1
#define KVSPACE_REF_EXT 2

/* storetype：物理布局维（head 第 3 字节，codec 唯一分派）。 */
#define KVSPACE_STORETYPE_NONE 0 /* 无物理字段，body 空 */
#define KVSPACE_STORETYPE_ATOM 1 /* 无物理字段，body 定宽 raw */
#define KVSPACE_STORETYPE_ARRAYND 2 /* 物理字段 ndim u8 + dims[ndim] u32 LE，body 稠密等宽数组 */
#define KVSPACE_STORETYPE_INDEX 3 /* 成员名矩阵 dims=[len,cap,M]（静态目录/值容器） */
#define KVSPACE_STORETYPE_EXTINDEX 4 /* 同 index，cap 可增长（运行栈等） */
/* Storage classes are the low two wire flag bits. */
#define KVSPACE_STORETYPE_FIXED_SMALL 0
#define KVSPACE_STORETYPE_SLACK 1
#define KVSPACE_STORETYPE_FIXED_LARGE 2
#define KVSPACE_STORETYPE_EXT 3

/* XValue 头（repr C）。三正交轴 ref/storetype/langtype；body 靠 headlen 定位。 */
/* Decoded XValue metadata. ro/vid come from the sidecar key on GetHead. */
typedef struct {
uint16_t headlen; /* head 总字节数；body 起于偏移 headlen */
uint8_t ref; /* 存储位置：见 KVSPACE_REF_* */
uint8_t storetype; /* 物理布局：见 KVSPACE_STORETYPE_* */
uint8_t storetype; /* Storage class. */
uint8_t ro; /* 1=只读，0=可写 */
uint32_t vid; /* vthread id（默认 0） */
int32_t body_len; /* body 字节数 */
int32_t ndim; /* ARRAYND：维数；index/extindex：3（[len,cap,M]）；NONE/ATOM：0 */
int32_t dims[8]; /* ARRAYND：各维长度；index/extindex：[len,cap,M] */
int32_t ndim; /* Tensor dimensions; zero for other values. */
int32_t dims[8]; /* Tensor shape. */
char langtype[256]; /* 语义类型 kindexpr 串，NUL 终止（含 [dims]、无 ptr/ext 前缀） */
int32_t langtype_len; /* langtype 内容长度（去 padding） */
int32_t body_offset; /* body 在 data 内的起始偏移（= headlen） */
uint64_t body_cap;
} kvspaceHead_t;

/* ── 生命周期 ─────────────────────────────────────────────────── */
Expand All @@ -71,6 +62,9 @@ void kvspaceClose(void *h);
/* 借用读：*out 指向后端常驻空间（shm mmap / durable 借用池），生命周期同该槽，
* 调用方不得 free。resolve=1 穿透 link。key 不存在/空值 → *out=NULL、*out_len=0、返回 0。 */
int kvspaceGet(void *h, const char *key, int resolve, uint8_t **out, uint32_t *out_len);
/* Store a complete XValue before returning. */
int kvspaceSetValue(void *h, const char *key, const uint8_t *value, uint32_t value_len,
uint8_t ro, uint32_t vid, char *err, uint32_t err_cap);

/* ResolveRef：block_id=叶子、gen=0；parent_id=目录祖先 ART 节点，
* depth=进入该节点时 key 已消费字节数。GetByRef：gen==0 直取叶子；
Expand Down Expand Up @@ -103,8 +97,7 @@ int kvspaceGetPart(void *h, const char *key, uint32_t offset, uint32_t len,
int kvspaceSetPart(void *h, const char *key, uint32_t offset, const uint8_t *buf,
uint32_t buf_len, char *err, uint32_t err_cap);

/* 读 head：只读值前缀并解码三正交轴 head（不取 body），供 xv 系列不借整块即得元数据。
* 空/不存在 → 返回非 0。 */
/* Read the wire head and ro/vid sidecar. Missing keys return nonzero. */
int kvspaceGetHead(void *h, const char *key, kvspaceHead_t *out);

/* 就地写：key 必须已存在、kind 不变、body_len 必须等于原 body_len——返回原 box 的 body
Expand All @@ -114,12 +107,10 @@ int kvspaceGetHead(void *h, const char *key, kvspaceHead_t *out);
int kvspaceWriteInPlace(void *h, const char *key, int resolve, uint32_t body_len,
uint8_t **body, char *err, uint32_t err_cap);

/* 新位置写：按 (ref, storetype, ro, vid, langtype, body_len) 分配新 box、写好 head，返回 body
* 偏移指针供直接写。ARRAYND 的 dims 由 codec 从 langtype 串内的 [dims] 解析落入物理字段。
* 用于新建 key 或 storetype/尺寸变化。落盘时机同 WriteInPlace：本笔在下一次 kvspace* 调用
* 或 kvspaceClose 时落盘。 */
/* Reserve a new XValue and return its body pointer; fill it before another KVSpace call. */
int kvspaceWriteNewPlace(void *h, const char *key, uint8_t ref, uint8_t storetype,
uint8_t ro, uint32_t vid, const char *langtype, uint32_t body_len,
uint64_t body_cap,
uint8_t **body, char *err, uint32_t err_cap);

/* 只返回前缀下子项计数，无缓冲、无需释放。resolve=1 穿透 link。 */
Expand Down
Loading
Loading