odb: embed base source in the "files" backend

The "files" backend is implemented as a pointer in the `struct
odb_source`. This contradicts our typical pattern for pluggable backends
like we use it for example in the ref store or for object database
streams, where we typically embed the generic base structure in the
specialized implementation. This pattern has a couple of small benefits:

  - We avoid an extra allocation.

  - We hide implementation details in the generic structure.

  - We can easily downcast from a generic backend to the specialized
    structure and vice versa because the offsets are known at compile
    time.

  - It becomes trivial to identify locations where we depend on backend
    specific logic because the cast needs to be explicit.

Refactor our "files" object database source to do the same and embed the
`struct odb_source` in the `struct odb_source_files`.

There are still a bunch of sites in our code base where we do have to
access internals of the "files" backend. The intent is that those will
go away over time, but this will certainly take a while. Meanwhile,
provide a `odb_source_files_downcast()` function that can convert a
generic source into a "files" source.

As we only have a single source the downcast succeeds unconditionally
for now. Eventually though the intent is to make the cast `BUG()` in
case the caller requests to downcast a non-"files" backend to a "files"
backend.

Signed-off-by: Patrick Steinhardt <ps@pks.im>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

Author: pks-t

builtin/cat-file.c

@@ -882,7 +882,8 @@ static void batch_each_object(struct batch_options *opt,
 		struct object_info oi = { 0 };
 
 		for (source = the_repository->objects->sources; source; source = source->next) {
-			int ret = packfile_store_for_each_object(source->files->packed, &oi,
+			struct odb_source_files *files = odb_source_files_downcast(source);
+			int ret = packfile_store_for_each_object(files->packed, &oi,
 								 batch_one_object_oi, &payload, flags);
 			if (ret)
 				break;

builtin/fast-import.c

@@ -875,6 +875,7 @@ static void end_packfile(void)
 	running = 1;
 	clear_delta_base_cache();
 	if (object_count) {
+		struct odb_source_files *files = odb_source_files_downcast(pack_data->repo->objects->sources);
 		struct packed_git *new_p;
 		struct object_id cur_pack_oid;
 		char *idx_name;
@@ -900,8 +901,7 @@ static void end_packfile(void)
 		idx_name = keep_pack(create_index());
 
 		/* Register the packfile with core git's machinery. */
-		new_p = packfile_store_load_pack(pack_data->repo->objects->sources->files->packed,
-						 idx_name, 1);
+		new_p = packfile_store_load_pack(files->packed, idx_name, 1);
 		if (!new_p)
 			die(_("core Git rejected index %s"), idx_name);
 		all_packs[pack_id] = new_p;
@@ -982,7 +982,9 @@ static int store_object(
 	}
 
 	for (source = the_repository->objects->sources; source; source = source->next) {
-		if (!packfile_list_find_oid(packfile_store_get_packs(source->files->packed), &oid))
+		struct odb_source_files *files = odb_source_files_downcast(source);
+
+		if (!packfile_list_find_oid(packfile_store_get_packs(files->packed), &oid))
 			continue;
 		e->type = type;
 		e->pack_id = MAX_PACK_ID;
@@ -1187,7 +1189,9 @@ static void stream_blob(uintmax_t len, struct object_id *oidout, uintmax_t mark)
 	}
 
 	for (source = the_repository->objects->sources; source; source = source->next) {
-		if (!packfile_list_find_oid(packfile_store_get_packs(source->files->packed), &oid))
+		struct odb_source_files *files = odb_source_files_downcast(source);
+
+		if (!packfile_list_find_oid(packfile_store_get_packs(files->packed), &oid))
 			continue;
 		e->type = OBJ_BLOB;
 		e->pack_id = MAX_PACK_ID;

builtin/grep.c

@@ -1218,8 +1218,10 @@ int cmd_grep(int argc,
 			struct odb_source *source;
 
 			odb_prepare_alternates(the_repository->objects);
-			for (source = the_repository->objects->sources; source; source = source->next)
-				packfile_store_prepare(source->files->packed);
+			for (source = the_repository->objects->sources; source; source = source->next) {
+				struct odb_source_files *files = odb_source_files_downcast(source);
+				packfile_store_prepare(files->packed);
+			}
 		}
 
 		start_threads(&opt);

builtin/index-pack.c

@@ -1637,9 +1637,11 @@ static void final(const char *final_pack_name, const char *curr_pack_name,
 	rename_tmp_packfile(&final_index_name, curr_index_name, &index_name,
 			    hash, "idx", 1);
 
-	if (do_fsck_object && startup_info->have_repository)
-		packfile_store_load_pack(the_repository->objects->sources->files->packed,
-					 final_index_name, 0);
+	if (do_fsck_object && startup_info->have_repository) {
+		struct odb_source_files *files =
+			odb_source_files_downcast(the_repository->objects->sources);
+		packfile_store_load_pack(files->packed, final_index_name, 0);
+	}
 
 	if (!from_stdin) {
 		printf("%s\n", hash_to_hex(hash));

builtin/pack-objects.c

@@ -1531,7 +1531,8 @@ static int want_cruft_object_mtime(struct repository *r,
 	struct odb_source *source;
 
 	for (source = r->objects->sources; source; source = source->next) {
-		struct packed_git **cache = packfile_store_get_kept_pack_cache(source->files->packed, flags);
+		struct odb_source_files *files = odb_source_files_downcast(source);
+		struct packed_git **cache = packfile_store_get_kept_pack_cache(files->packed, flags);
 
 		for (; *cache; cache++) {
 			struct packed_git *p = *cache;
@@ -1753,11 +1754,13 @@ static int want_object_in_pack_mtime(const struct object_id *oid,
 	}
 
 	for (source = the_repository->objects->sources; source; source = source->next) {
-		for (e = source->files->packed->packs.head; e; e = e->next) {
+		struct odb_source_files *files = odb_source_files_downcast(source);
+
+		for (e = files->packed->packs.head; e; e = e->next) {
 			struct packed_git *p = e->pack;
 			want = want_object_in_pack_one(p, oid, exclude, found_pack, found_offset, found_mtime);
 			if (!exclude && want > 0)
-				packfile_list_prepend(&source->files->packed->packs, p);
+				packfile_list_prepend(&files->packed->packs, p);
 			if (want != -1)
 				return want;
 		}
@@ -4337,10 +4340,12 @@ static void add_objects_in_unpacked_packs(void)
 
 	odb_prepare_alternates(to_pack.repo->objects);
 	for (source = to_pack.repo->objects->sources; source; source = source->next) {
+		struct odb_source_files *files = odb_source_files_downcast(source);
+
 		if (!source->local)
 			continue;
 
-		if (packfile_store_for_each_object(source->files->packed, &oi,
+		if (packfile_store_for_each_object(files->packed, &oi,
 						   add_object_in_unpacked_pack, NULL,
 						   ODB_FOR_EACH_OBJECT_PACK_ORDER |
 						   ODB_FOR_EACH_OBJECT_LOCAL_ONLY |

commit-graph.c

@@ -1980,9 +1980,11 @@ static void fill_oids_from_all_packs(struct write_commit_graph_context *ctx)
 			ctx->approx_nr_objects);
 
 	odb_prepare_alternates(ctx->r->objects);
-	for (source = ctx->r->objects->sources; source; source = source->next)
-		packfile_store_for_each_object(source->files->packed, &oi, add_packed_commits_oi,
+	for (source = ctx->r->objects->sources; source; source = source->next) {
+		struct odb_source_files *files = odb_source_files_downcast(source);
+		packfile_store_for_each_object(files->packed, &oi, add_packed_commits_oi,
 					       ctx, ODB_FOR_EACH_OBJECT_PACK_ORDER);
+	}
 
 	if (ctx->progress_done < ctx->approx_nr_objects)
 		display_progress(ctx->progress, ctx->approx_nr_objects);

http.c

@@ -2543,8 +2543,9 @@ int finish_http_pack_request(struct http_pack_request *preq)
 void http_install_packfile(struct packed_git *p,
 			   struct packfile_list *list_to_remove_from)
 {
+	struct odb_source_files *files = odb_source_files_downcast(the_repository->objects->sources);
 	packfile_list_remove(list_to_remove_from, p);
-	packfile_store_add_pack(the_repository->objects->sources->files->packed, p);
+	packfile_store_add_pack(files->packed, p);
 }
 
 struct http_pack_request *new_http_pack_request(

loose.c

@@ -3,6 +3,7 @@
 #include "path.h"
 #include "object-file.h"
 #include "odb.h"
+#include "odb/source-files.h"
 #include "hex.h"
 #include "repository.h"
 #include "wrapper.h"
@@ -49,27 +50,29 @@ static int insert_loose_map(struct odb_source *source,
 			    const struct object_id *oid,
 			    const struct object_id *compat_oid)
 {
-	struct loose_object_map *map = source->files->loose->map;
+	struct odb_source_files *files = odb_source_files_downcast(source);
+	struct loose_object_map *map = files->loose->map;
 	int inserted = 0;
 
 	inserted |= insert_oid_pair(map->to_compat, oid, compat_oid);
 	inserted |= insert_oid_pair(map->to_storage, compat_oid, oid);
 	if (inserted)
-		oidtree_insert(source->files->loose->cache, compat_oid);
+		oidtree_insert(files->loose->cache, compat_oid);
 
 	return inserted;
 }
 
 static int load_one_loose_object_map(struct repository *repo, struct odb_source *source)
 {
+	struct odb_source_files *files = odb_source_files_downcast(source);
 	struct strbuf buf = STRBUF_INIT, path = STRBUF_INIT;
 	FILE *fp;
 
-	if (!source->files->loose->map)
-		loose_object_map_init(&source->files->loose->map);
-	if (!source->files->loose->cache) {
-		ALLOC_ARRAY(source->files->loose->cache, 1);
-		oidtree_init(source->files->loose->cache);
+	if (!files->loose->map)
+		loose_object_map_init(&files->loose->map);
+	if (!files->loose->cache) {
+		ALLOC_ARRAY(files->loose->cache, 1);
+		oidtree_init(files->loose->cache);
 	}
 
 	insert_loose_map(source, repo->hash_algo->empty_tree, repo->compat_hash_algo->empty_tree);
@@ -125,7 +128,8 @@ int repo_read_loose_object_map(struct repository *repo)
 
 int repo_write_loose_object_map(struct repository *repo)
 {
-	kh_oid_map_t *map = repo->objects->sources->files->loose->map->to_compat;
+	struct odb_source_files *files = odb_source_files_downcast(repo->objects->sources);
+	kh_oid_map_t *map = files->loose->map->to_compat;
 	struct lock_file lock;
 	int fd;
 	khiter_t iter;
@@ -231,7 +235,8 @@ int repo_loose_object_map_oid(struct repository *repo,
 	khiter_t pos;
 
 	for (source = repo->objects->sources; source; source = source->next) {
-		struct loose_object_map *loose_map = source->files->loose->map;
+		struct odb_source_files *files = odb_source_files_downcast(source);
+		struct loose_object_map *loose_map = files->loose->map;
 		if (!loose_map)
 			continue;
 		map = (to == repo->compat_hash_algo) ?

midx.c

@@ -95,8 +95,9 @@ static int midx_read_object_offsets(const unsigned char *chunk_start,
 
 struct multi_pack_index *get_multi_pack_index(struct odb_source *source)
 {
-	packfile_store_prepare(source->files->packed);
-	return source->files->packed->midx;
+	struct odb_source_files *files = odb_source_files_downcast(source);
+	packfile_store_prepare(files->packed);
+	return files->packed->midx;
 }
 
 static struct multi_pack_index *load_multi_pack_index_one(struct odb_source *source,
@@ -447,6 +448,7 @@ static uint32_t midx_for_pack(struct multi_pack_index **_m,
 int prepare_midx_pack(struct multi_pack_index *m,
 		      uint32_t pack_int_id)
 {
+	struct odb_source_files *files = odb_source_files_downcast(m->source);
 	struct strbuf pack_name = STRBUF_INIT;
 	struct packed_git *p;
 
@@ -457,10 +459,10 @@ int prepare_midx_pack(struct multi_pack_index *m,
 	if (m->packs[pack_int_id])
 		return 0;
 
-	strbuf_addf(&pack_name, "%s/pack/%s", m->source->path,
+	strbuf_addf(&pack_name, "%s/pack/%s", files->base.path,
 		    m->pack_names[pack_int_id]);
-	p = packfile_store_load_pack(m->source->files->packed,
-				     pack_name.buf, m->source->local);
+	p = packfile_store_load_pack(files->packed,
+				     pack_name.buf, files->base.local);
 	strbuf_release(&pack_name);
 
 	if (!p) {
@@ -703,18 +705,19 @@ int midx_preferred_pack(struct multi_pack_index *m, uint32_t *pack_int_id)
 
 int prepare_multi_pack_index_one(struct odb_source *source)
 {
+	struct odb_source_files *files = odb_source_files_downcast(source);
 	struct repository *r = source->odb->repo;
 
 	prepare_repo_settings(r);
 	if (!r->settings.core_multi_pack_index)
 		return 0;
 
-	if (source->files->packed->midx)
+	if (files->packed->midx)
 		return 1;
 
-	source->files->packed->midx = load_multi_pack_index(source);
+	files->packed->midx = load_multi_pack_index(source);
 
-	return !!source->files->packed->midx;
+	return !!files->packed->midx;
 }
 
 int midx_checksum_valid(struct multi_pack_index *m)
@@ -803,9 +806,10 @@ void clear_midx_file(struct repository *r)
 		struct odb_source *source;
 
 		for (source = r->objects->sources; source; source = source->next) {
-			if (source->files->packed->midx)
-				close_midx(source->files->packed->midx);
-			source->files->packed->midx = NULL;
+			struct odb_source_files *files = odb_source_files_downcast(source);
+			if (files->packed->midx)
+				close_midx(files->packed->midx);
+			files->packed->midx = NULL;
 		}
 	}
 

object-file.c

@@ -219,8 +219,9 @@ static void *odb_source_loose_map_object(struct odb_source *source,
 					 const struct object_id *oid,
 					 unsigned long *size)
 {
+	struct odb_source_files *files = odb_source_files_downcast(source);
 	const char *p;
-	int fd = open_loose_object(source->files->loose, oid, &p);
+	int fd = open_loose_object(files->loose, oid, &p);
 
 	if (fd < 0)
 		return NULL;
@@ -401,6 +402,7 @@ static int read_object_info_from_path(struct odb_source *source,
 				      struct object_info *oi,
 				      enum object_info_flags flags)
 {
+	struct odb_source_files *files = odb_source_files_downcast(source);
 	int ret;
 	int fd;
 	unsigned long mapsize;
@@ -423,7 +425,7 @@ static int read_object_info_from_path(struct odb_source *source,
 		struct stat st;
 
 		if ((!oi || (!oi->disk_sizep && !oi->mtimep)) && (flags & OBJECT_INFO_QUICK)) {
-			ret = quick_has_loose(source->files->loose, oid) ? 0 : -1;
+			ret = quick_has_loose(files->loose, oid) ? 0 : -1;
 			goto out;
 		}
 
@@ -1866,33 +1868,34 @@ static int append_loose_object(const struct object_id *oid,
 struct oidtree *odb_source_loose_cache(struct odb_source *source,
 				       const struct object_id *oid)
 {
+	struct odb_source_files *files = odb_source_files_downcast(source);
 	int subdir_nr = oid->hash[0];
 	struct strbuf buf = STRBUF_INIT;
-	size_t word_bits = bitsizeof(source->files->loose->subdir_seen[0]);
+	size_t word_bits = bitsizeof(files->loose->subdir_seen[0]);
 	size_t word_index = subdir_nr / word_bits;
 	size_t mask = (size_t)1u << (subdir_nr % word_bits);
 	uint32_t *bitmap;
 
 	if (subdir_nr < 0 ||
-	    (size_t) subdir_nr >= bitsizeof(source->files->loose->subdir_seen))
+	    (size_t) subdir_nr >= bitsizeof(files->loose->subdir_seen))
 		BUG("subdir_nr out of range");
 
-	bitmap = &source->files->loose->subdir_seen[word_index];
+	bitmap = &files->loose->subdir_seen[word_index];
 	if (*bitmap & mask)
-		return source->files->loose->cache;
-	if (!source->files->loose->cache) {
-		ALLOC_ARRAY(source->files->loose->cache, 1);
-		oidtree_init(source->files->loose->cache);
+		return files->loose->cache;
+	if (!files->loose->cache) {
+		ALLOC_ARRAY(files->loose->cache, 1);
+		oidtree_init(files->loose->cache);
 	}
 	strbuf_addstr(&buf, source->path);
 	for_each_file_in_obj_subdir(subdir_nr, &buf,
 				    source->odb->repo->hash_algo,
 				    append_loose_object,
 				    NULL, NULL,
-				    source->files->loose->cache);
+				    files->loose->cache);
 	*bitmap |= mask;
 	strbuf_release(&buf);
-	return source->files->loose->cache;
+	return files->loose->cache;
 }
 
 static void odb_source_loose_clear_cache(struct odb_source_loose *loose)
@@ -1905,7 +1908,8 @@ static void odb_source_loose_clear_cache(struct odb_source_loose *loose)
 
 void odb_source_loose_reprepare(struct odb_source *source)
 {
-	odb_source_loose_clear_cache(source->files->loose);
+	struct odb_source_files *files = odb_source_files_downcast(source);
+	odb_source_loose_clear_cache(files->loose);
 }
 
 static int check_stream_oid(git_zstream *stream,