Merge branch 'ps/reftable-portability'

gitster · gitster · commit 4fee6ff3b233 · 2026-04-08T10:19:17.000-07:00
Update reftable library part with what is used in libgit2 to improve
portability to different target codebases and platforms.

* ps/reftable-portability:
  reftable/system: add abstraction to mmap files
  reftable/system: add abstraction to retrieve time in milliseconds
  reftable/fsck: use REFTABLE_UNUSED instead of UNUSED
  reftable/stack: provide fsync(3p) via system header
  reftable: introduce "reftable-system.h" header
diff --git a/refs/reftable-backend.c b/refs/reftable-backend.c
@@ -366,11 +366,6 @@ static int reftable_be_config(const char *var, const char *value,
 	return 0;
 }
 
-static int reftable_be_fsync(int fd)
-{
-	return fsync_component(FSYNC_COMPONENT_REFERENCE, fd);
-}
-
 static struct ref_store *reftable_be_init(struct repository *repo,
 					  const char *payload,
 					  const char *gitdir,
@@ -408,7 +403,6 @@ static struct ref_store *reftable_be_init(struct repository *repo,
 	refs->write_options.disable_auto_compact =
 		!git_env_bool("GIT_TEST_REFTABLE_AUTOCOMPACTION", 1);
 	refs->write_options.lock_timeout_ms = 100;
-	refs->write_options.fsync = reftable_be_fsync;
 
 	repo_config(the_repository, reftable_be_config, &refs->write_options);
 
diff --git a/reftable/blocksource.c b/reftable/blocksource.c
@@ -93,13 +93,12 @@ void block_source_from_buf(struct reftable_block_source *bs,
 }
 
 struct file_block_source {
-	uint64_t size;
-	unsigned char *data;
+	struct reftable_mmap mmap;
 };
 
 static uint64_t file_size(void *b)
 {
-	return ((struct file_block_source *)b)->size;
+	return ((struct file_block_source *)b)->mmap.size;
 }
 
 static void file_release_data(void *b REFTABLE_UNUSED, struct reftable_block_data *dest REFTABLE_UNUSED)
@@ -109,16 +108,16 @@ static void file_release_data(void *b REFTABLE_UNUSED, struct reftable_block_dat
 static void file_close(void *v)
 {
 	struct file_block_source *b = v;
-	munmap(b->data, b->size);
+	reftable_munmap(&b->mmap);
 	reftable_free(b);
 }
 
 static ssize_t file_read_data(void *v, struct reftable_block_data *dest, uint64_t off,
 			      uint32_t size)
 {
 	struct file_block_source *b = v;
-	assert(off + size <= b->size);
-	dest->data = b->data + off;
+	assert(off + size <= b->mmap.size);
+	dest->data = (unsigned char *) b->mmap.data + off;
 	dest->len = size;
 	return size;
 }
@@ -156,13 +155,9 @@ int reftable_block_source_from_file(struct reftable_block_source *bs,
 		goto out;
 	}
 
-	p->size = st.st_size;
-	p->data = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
-	if (p->data == MAP_FAILED) {
-		err = REFTABLE_IO_ERROR;
-		p->data = NULL;
+	err = reftable_mmap(&p->mmap, fd, st.st_size);
+	if (err < 0)
 		goto out;
-	}
 
 	assert(!bs->ops);
 	bs->ops = &file_vtable;
diff --git a/reftable/fsck.c b/reftable/fsck.c
@@ -63,7 +63,7 @@ static int table_check_name(struct reftable_table *table,
 
 static int table_checks(struct reftable_table *table,
 			reftable_fsck_report_fn report_fn,
-			reftable_fsck_verbose_fn verbose_fn UNUSED,
+			reftable_fsck_verbose_fn verbose_fn REFTABLE_UNUSED,
 			void *cb_data)
 {
 	table_check_fn table_check_fns[] = {
diff --git a/reftable/reftable-basics.h b/reftable/reftable-basics.h
@@ -9,7 +9,7 @@
 #ifndef REFTABLE_BASICS_H
 #define REFTABLE_BASICS_H
 
-#include <stddef.h>
+#include "reftable-system.h"
 
 /* A buffer that contains arbitrary byte slices. */
 struct reftable_buf {
diff --git a/reftable/reftable-block.h b/reftable/reftable-block.h
@@ -9,8 +9,7 @@
 #ifndef REFTABLE_BLOCK_H
 #define REFTABLE_BLOCK_H
 
-#include <stdint.h>
-
+#include "reftable-system.h"
 #include "reftable-basics.h"
 #include "reftable-blocksource.h"
 #include "reftable-iterator.h"
diff --git a/reftable/reftable-blocksource.h b/reftable/reftable-blocksource.h
@@ -9,7 +9,7 @@
 #ifndef REFTABLE_BLOCKSOURCE_H
 #define REFTABLE_BLOCKSOURCE_H
 
-#include <stdint.h>
+#include "reftable-system.h"
 
 /*
  * Generic wrapper for a seekable readable file.
diff --git a/reftable/reftable-error.h b/reftable/reftable-error.h
@@ -9,6 +9,8 @@
 #ifndef REFTABLE_ERROR_H
 #define REFTABLE_ERROR_H
 
+#include "reftable-system.h"
+
 /*
  * Errors in reftable calls are signaled with negative integer return values. 0
  * means success.
diff --git a/reftable/reftable-fsck.h b/reftable/reftable-fsck.h
@@ -1,6 +1,7 @@
 #ifndef REFTABLE_FSCK_H
 #define REFTABLE_FSCK_H
 
+#include "reftable-system.h"
 #include "reftable-stack.h"
 
 enum reftable_fsck_error {
diff --git a/reftable/reftable-iterator.h b/reftable/reftable-iterator.h
@@ -9,6 +9,7 @@
 #ifndef REFTABLE_ITERATOR_H
 #define REFTABLE_ITERATOR_H
 
+#include "reftable-system.h"
 #include "reftable-record.h"
 
 struct reftable_iterator_vtable;
diff --git a/reftable/reftable-merged.h b/reftable/reftable-merged.h
@@ -9,6 +9,7 @@
 #ifndef REFTABLE_MERGED_H
 #define REFTABLE_MERGED_H
 
+#include "reftable-system.h"
 #include "reftable-iterator.h"
 
 /*
diff --git a/reftable/reftable-record.h b/reftable/reftable-record.h
@@ -9,8 +9,8 @@
 #ifndef REFTABLE_RECORD_H
 #define REFTABLE_RECORD_H
 
+#include "reftable-system.h"
 #include "reftable-basics.h"
-#include <stdint.h>
 
 /*
  * Basic data types
diff --git a/reftable/reftable-stack.h b/reftable/reftable-stack.h
@@ -9,6 +9,7 @@
 #ifndef REFTABLE_STACK_H
 #define REFTABLE_STACK_H
 
+#include "reftable-system.h"
 #include "reftable-writer.h"
 
 /*
diff --git a/reftable/reftable-system.h b/reftable/reftable-system.h
@@ -0,0 +1,18 @@
+#ifndef REFTABLE_SYSTEM_H
+#define REFTABLE_SYSTEM_H
+
+/*
+ * This header defines the platform-specific bits required to compile the
+ * reftable library. It should provide an environment that bridges over the
+ * gaps between POSIX and your system, as well as the zlib interfaces. This
+ * header is expected to be changed by the individual project.
+ */
+
+#define MINGW_DONT_HANDLE_IN_USE_ERROR
+#include "compat/posix.h"
+#include "compat/zlib-compat.h"
+
+int reftable_fsync(int fd);
+#define fsync(fd) reftable_fsync(fd)
+
+#endif
diff --git a/reftable/reftable-table.h b/reftable/reftable-table.h
@@ -9,6 +9,7 @@
 #ifndef REFTABLE_TABLE_H
 #define REFTABLE_TABLE_H
 
+#include "reftable-system.h"
 #include "reftable-iterator.h"
 #include "reftable-block.h"
 #include "reftable-blocksource.h"
diff --git a/reftable/reftable-writer.h b/reftable/reftable-writer.h
@@ -9,11 +9,9 @@
 #ifndef REFTABLE_WRITER_H
 #define REFTABLE_WRITER_H
 
+#include "reftable-system.h"
 #include "reftable-record.h"
 
-#include <stdint.h>
-#include <unistd.h> /* ssize_t */
-
 /* Writing single reftables */
 
 /* reftable_write_options sets options for writing a single reftable. */
@@ -63,12 +61,6 @@ struct reftable_write_options {
 	 */
 	long lock_timeout_ms;
 
-	/*
-	 * Optional callback used to fsync files to disk. Falls back to using
-	 * fsync(3P) when unset.
-	 */
-	int (*fsync)(int fd);
-
 	/*
 	 * Callback function to execute whenever the stack is being reloaded.
 	 * This can be used e.g. to discard cached information that relies on
diff --git a/reftable/stack.c b/reftable/stack.c
@@ -29,13 +29,6 @@ static int stack_filename(struct reftable_buf *dest, struct reftable_stack *st,
 	return 0;
 }
 
-static int stack_fsync(const struct reftable_write_options *opts, int fd)
-{
-	if (opts->fsync)
-		return opts->fsync(fd);
-	return fsync(fd);
-}
-
 static ssize_t reftable_write_data(int fd, const void *data, size_t size)
 {
 	size_t total_written = 0;
@@ -69,7 +62,7 @@ static ssize_t fd_writer_write(void *arg, const void *data, size_t sz)
 static int fd_writer_flush(void *arg)
 {
 	struct fd_writer *writer = arg;
-	return stack_fsync(writer->opts, writer->fd);
+	return fsync(writer->fd);
 }
 
 static int fd_read_lines(int fd, char ***namesp)
@@ -372,45 +365,26 @@ static int reftable_stack_reload_once(struct reftable_stack *st,
 	return err;
 }
 
-/* return negative if a before b. */
-static int tv_cmp(struct timeval *a, struct timeval *b)
-{
-	time_t diff = a->tv_sec - b->tv_sec;
-	int udiff = a->tv_usec - b->tv_usec;
-
-	if (diff != 0)
-		return diff;
-
-	return udiff;
-}
-
 static int reftable_stack_reload_maybe_reuse(struct reftable_stack *st,
 					     int reuse_open)
 {
 	char **names = NULL, **names_after = NULL;
-	struct timeval deadline;
+	uint64_t deadline;
 	int64_t delay = 0;
 	int tries = 0, err;
 	int fd = -1;
 
-	err = gettimeofday(&deadline, NULL);
-	if (err < 0)
-		goto out;
-	deadline.tv_sec += 3;
+	deadline = reftable_time_ms() + 3000;
 
 	while (1) {
-		struct timeval now;
-
-		err = gettimeofday(&now, NULL);
-		if (err < 0)
-			goto out;
+		uint64_t now = reftable_time_ms();
 
 		/*
 		 * Only look at deadlines after the first few times. This
 		 * simplifies debugging in GDB.
 		 */
 		tries++;
-		if (tries > 3 && tv_cmp(&now, &deadline) >= 0)
+		if (tries > 3 && now >= deadline)
 			goto out;
 
 		fd = open(st->list_file, O_RDONLY);
@@ -812,7 +786,7 @@ int reftable_addition_commit(struct reftable_addition *add)
 		goto done;
 	}
 
-	err = stack_fsync(&add->stack->opts, add->tables_list_lock.fd);
+	err = fsync(add->tables_list_lock.fd);
 	if (err < 0) {
 		err = REFTABLE_IO_ERROR;
 		goto done;
@@ -1480,7 +1454,7 @@ static int stack_compact_range(struct reftable_stack *st,
 		goto done;
 	}
 
-	err = stack_fsync(&st->opts, tables_list_lock.fd);
+	err = fsync(tables_list_lock.fd);
 	if (err < 0) {
 		err = REFTABLE_IO_ERROR;
 		unlink(new_table_path.buf);
diff --git a/reftable/system.c b/reftable/system.c
@@ -4,7 +4,9 @@
 #include "basics.h"
 #include "reftable-error.h"
 #include "../lockfile.h"
+#include "../trace.h"
 #include "../tempfile.h"
+#include "../write-or-die.h"
 
 uint32_t reftable_rand(void)
 {
@@ -131,3 +133,33 @@ int flock_commit(struct reftable_flock *l)
 
 	return 0;
 }
+
+int reftable_fsync(int fd)
+{
+	return fsync_component(FSYNC_COMPONENT_REFERENCE, fd);
+}
+
+uint64_t reftable_time_ms(void)
+{
+	return getnanotime() / 1000000;
+}
+
+int reftable_mmap(struct reftable_mmap *out, int fd, size_t len)
+{
+	void *data = xmmap_gently(NULL, len, PROT_READ, MAP_PRIVATE, fd, 0);
+	if (data == MAP_FAILED)
+		return REFTABLE_IO_ERROR;
+
+	out->data = data;
+	out->size = len;
+
+	return 0;
+}
+
+int reftable_munmap(struct reftable_mmap *mmap)
+{
+	if (munmap(mmap->data, mmap->size) < 0)
+		return REFTABLE_IO_ERROR;
+	memset(mmap, 0, sizeof(*mmap));
+	return 0;
+}
diff --git a/reftable/system.h b/reftable/system.h
@@ -9,11 +9,14 @@
 #ifndef SYSTEM_H
 #define SYSTEM_H
 
-/* This header glues the reftable library to the rest of Git */
+/*
+ * This header defines the platform-agnostic interface that is to be
+ * implemented by the project to make it work on their respective supported
+ * systems, and to integrate it into the project itself. This header is not
+ * expected to be changed by the individual project.
+ */
 
-#define MINGW_DONT_HANDLE_IN_USE_ERROR
-#include "compat/posix.h"
-#include "compat/zlib-compat.h"
+#include "reftable-system.h"
 
 /*
  * Return a random 32 bit integer. This function is expected to return
@@ -108,4 +111,25 @@ int flock_release(struct reftable_flock *l);
  */
 int flock_commit(struct reftable_flock *l);
 
+/* Report the time in milliseconds. */
+uint64_t reftable_time_ms(void);
+
+struct reftable_mmap {
+       void *data;
+       size_t size;
+       void *priv;
+};
+
+/*
+ * Map the file into memory. Returns 0 on success, a reftable error code on
+ * error.
+ */
+int reftable_mmap(struct reftable_mmap *out, int fd, size_t len);
+
+/*
+ * Unmap the file from memory. Returns 0 on success, a reftable error code on
+ * error.
+ */
+int reftable_munmap(struct reftable_mmap *mmap);
+
 #endif

Original file line number	Diff line number	Diff line change
`@@ -93,13 +93,12 @@ void block_source_from_buf(struct reftable_block_source *bs,`
`93`	`93`	`}`
`94`	`94`
`95`	`95`	`struct file_block_source {`
`96`		`- uint64_t size;`
`97`		`- unsigned char *data;`
	`96`	`+ struct reftable_mmap mmap;`
`98`	`97`	`};`
`99`	`98`
`100`	`99`	`static uint64_t file_size(void *b)`
`101`	`100`	`{`
`102`		`- return ((struct file_block_source *)b)->size;`
	`101`	`+ return ((struct file_block_source *)b)->mmap.size;`
`103`	`102`	`}`
`104`	`103`
`105`	`104`	`static void file_release_data(void b REFTABLE_UNUSED, struct reftable_block_data dest REFTABLE_UNUSED)`
`@@ -109,16 +108,16 @@ static void file_release_data(void *b REFTABLE_UNUSED, struct reftable_block_dat`
`109`	`108`	`static void file_close(void *v)`
`110`	`109`	`{`
`111`	`110`	`struct file_block_source *b = v;`
`112`		`- munmap(b->data, b->size);`
	`111`	`+ reftable_munmap(&b->mmap);`
`113`	`112`	`reftable_free(b);`
`114`	`113`	`}`
`115`	`114`
`116`	`115`	`static ssize_t file_read_data(void v, struct reftable_block_data dest, uint64_t off,`
`117`	`116`	`uint32_t size)`
`118`	`117`	`{`
`119`	`118`	`struct file_block_source *b = v;`
`120`		`- assert(off + size <= b->size);`
`121`		`- dest->data = b->data + off;`
	`119`	`+ assert(off + size <= b->mmap.size);`
	`120`	`+ dest->data = (unsigned char *) b->mmap.data + off;`
`122`	`121`	`dest->len = size;`
`123`	`122`	`return size;`
`124`	`123`	`}`
`@@ -156,13 +155,9 @@ int reftable_block_source_from_file(struct reftable_block_source *bs,`
`156`	`155`	`goto out;`
`157`	`156`	`}`
`158`	`157`
`159`		`- p->size = st.st_size;`
`160`		`- p->data = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);`
`161`		`- if (p->data == MAP_FAILED) {`
`162`		`- err = REFTABLE_IO_ERROR;`
`163`		`- p->data = NULL;`
	`158`	`+ err = reftable_mmap(&p->mmap, fd, st.st_size);`
	`159`	`+ if (err < 0)`
`164`	`160`	`goto out;`
`165`		`- }`
`166`	`161`
`167`	`162`	`assert(!bs->ops);`
`168`	`163`	`bs->ops = &file_vtable;`
Original file line number	Diff line number	Diff line change
`@@ -63,7 +63,7 @@ static int table_check_name(struct reftable_table *table,`
`63`	`63`
`64`	`64`	`static int table_checks(struct reftable_table *table,`
`65`	`65`	`reftable_fsck_report_fn report_fn,`
`66`		`- reftable_fsck_verbose_fn verbose_fn UNUSED,`
	`66`	`+ reftable_fsck_verbose_fn verbose_fn REFTABLE_UNUSED,`
`67`	`67`	`void *cb_data)`
`68`	`68`	`{`
`69`	`69`	`table_check_fn table_check_fns[] = {`