From 9a217f2a725b085982e57b2a900d46128713cb27 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Tue, 28 Jun 2005 14:56:57 -0700 Subject: [PATCH 1/2] [PATCH] Expose packed_git and alt_odb. The commands git-fsck-cache and probably git-*-pull needs to have a way to enumerate objects contained in packed GIT archives and alternate object pools. This commit exposes the data structure used to keep track of them from sha1_file.c, and adds a couple of accessor interface functions for use by the enhanced git-fsck-cache command. Signed-off-by: Junio C Hamano Signed-off-by: Linus Torvalds --- cache.h | 19 +++++++++++++++++++ sha1_file.c | 43 ++++++++++++++++++++++++------------------- 2 files changed, 43 insertions(+), 19 deletions(-) diff --git a/cache.h b/cache.h index 383fc866416..9bfcdb641a9 100644 --- a/cache.h +++ b/cache.h @@ -233,4 +233,23 @@ struct checkout { extern int checkout_entry(struct cache_entry *ce, struct checkout *state); +extern struct alternate_object_database { + char *base; + char *name; +} *alt_odb; +extern void prepare_alt_odb(void); + +extern struct packed_git { + struct packed_git *next; + unsigned long index_size; + unsigned long pack_size; + unsigned int *index_base; + void *pack_base; + unsigned int pack_last_used; + char pack_name[0]; /* something like ".git/objects/pack/xxxxx.pack" */ +} *packed_git; +extern void prepare_packed_git(void); +extern int num_packed_objects(const struct packed_git *p); +extern int nth_packed_object_sha1(const struct packed_git *, int, unsigned char*); + #endif /* CACHE_H */ diff --git a/sha1_file.c b/sha1_file.c index e27affb2885..17546061c15 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -184,10 +184,7 @@ char *sha1_file_name(const unsigned char *sha1) return base; } -static struct alternate_object_database { - char *base; - char *name; -} *alt_odb; +struct alternate_object_database *alt_odb; /* * Prepare alternate object database registry. @@ -205,13 +202,15 @@ static struct alternate_object_database { * pointed by base fields of the array elements with one xmalloc(); * the string pool immediately follows the array. */ -static void prepare_alt_odb(void) +void prepare_alt_odb(void) { int pass, totlen, i; const char *cp, *last; char *op = NULL; const char *alt = gitenv(ALTERNATE_DB_ENVIRONMENT) ? : ""; + if (alt_odb) + return; /* The first pass counts how large an area to allocate to * hold the entire alt_odb structure, including array of * structs and path buffers for them. The second pass fills @@ -258,8 +257,7 @@ static char *find_sha1_file(const unsigned char *sha1, struct stat *st) if (!stat(name, st)) return name; - if (!alt_odb) - prepare_alt_odb(); + prepare_alt_odb(); for (i = 0; (name = alt_odb[i].name) != NULL; i++) { fill_sha1_path(name, sha1); if (!stat(alt_odb[i].base, st)) @@ -271,15 +269,7 @@ static char *find_sha1_file(const unsigned char *sha1, struct stat *st) #define PACK_MAX_SZ (1<<26) static int pack_used_ctr; static unsigned long pack_mapped; -static struct packed_git { - struct packed_git *next; - unsigned long index_size; - unsigned long pack_size; - unsigned int *index_base; - void *pack_base; - unsigned int pack_last_used; - char pack_name[0]; /* something like ".git/objects/pack/xxxxx.pack" */ -} *packed_git; +struct packed_git *packed_git; struct pack_entry { unsigned int offset; @@ -429,7 +419,7 @@ static void prepare_packed_git_one(char *objdir) } } -static void prepare_packed_git(void) +void prepare_packed_git(void) { int i; static int run_once = 0; @@ -438,8 +428,7 @@ static void prepare_packed_git(void) return; prepare_packed_git_one(get_object_directory()); - if (!alt_odb) - prepare_alt_odb(); + prepare_alt_odb(); for (i = 0; alt_odb[i].base != NULL; i++) { alt_odb[i].name[0] = 0; prepare_packed_git_one(alt_odb[i].base); @@ -820,6 +809,22 @@ static void *unpack_entry(struct pack_entry *entry, return unpack_non_delta_entry(pack+5, size, left); } +int num_packed_objects(const struct packed_git *p) +{ + /* See check_packed_git_idx and pack-objects.c */ + return (p->index_size - 20 - 20 - 4*256) / 24; +} + +int nth_packed_object_sha1(const struct packed_git *p, int n, + unsigned char* sha1) +{ + void *index = p->index_base + 256; + if (n < 0 || num_packed_objects(p) <= n) + return -1; + memcpy(sha1, (index + 24 * n + 4), 20); + return 0; +} + static int find_pack_entry_1(const unsigned char *sha1, struct pack_entry *e, struct packed_git *p) { From 8a498a05c3c6b2f53db669b24f36257ab213eb4c Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Tue, 28 Jun 2005 14:58:33 -0700 Subject: [PATCH 2/2] [PATCH] Update fsck-cache (take 2) The fsck-cache complains if objects referred to by files in .git/refs/ or objects stored in files under .git/objects/??/ are not found as stand-alone SHA1 files (i.e. found in alternate object pools GIT_ALTERNATE_OBJECT_DIRECTORIES or packed archives stored under .git/objects/pack). Although this is a good semantics to maintain consistency of a single .git/objects directory as a self contained set of objects, it sometimes is useful to consider it is OK as long as these "outside" objects are available. This commit introduces a new flag, --standalone, to git-fsck-cache. When it is not specified, connectivity checks and .git/refs pointer checks are taught that it is OK when expected objects do not exist under .git/objects/?? hierarchy but are available from an packed archive or in an alternate object pool. Another new flag, --full, makes git-fsck-cache to check not only the current GIT_OBJECT_DIRECTORY but also objects found in alternate object pools and packed GIT archives.a Signed-off-by: Junio C Hamano Signed-off-by: Linus Torvalds --- Documentation/git-fsck-cache.txt | 18 +++++++- fsck-cache.c | 71 ++++++++++++++++++++++++++------ 2 files changed, 76 insertions(+), 13 deletions(-) diff --git a/Documentation/git-fsck-cache.txt b/Documentation/git-fsck-cache.txt index 0ef01c3ffcf..f1c18c10d82 100644 --- a/Documentation/git-fsck-cache.txt +++ b/Documentation/git-fsck-cache.txt @@ -9,7 +9,7 @@ git-fsck-cache - Verifies the connectivity and validity of the objects in the da SYNOPSIS -------- -'git-fsck-cache' [--tags] [--root] [--unreachable] [--cache] [*] +'git-fsck-cache' [--tags] [--root] [--unreachable] [--cache] [--standalone | --full] [*] DESCRIPTION ----------- @@ -37,6 +37,22 @@ OPTIONS Consider any object recorded in the cache also as a head node for an unreachability trace. +--standalone:: + Limit checks to the contents of GIT_OBJECT_DIRECTORY + (.git/objects), making sure that it is consistent and + complete without referring to objects found in alternate + object pools listed in GIT_ALTERNATE_OBJECT_DIRECTORIES, + nor packed GIT archives found in .git/objects/pack; + cannot be used with --full. + +--full:: + Check not just objects in GIT_OBJECT_DIRECTORY + (.git/objects), but also the ones found in alternate + object pools listed in GIT_ALTERNATE_OBJECT_DIRECTORIES, + and in packed GIT archives found in .git/objects/pack + and corresponding pack subdirectories in alternate + object pools; cannot be used with --standalone. + It tests SHA1 and general object sanity, and it does full tracking of the resulting reachability and everything else. It prints out any corruption it finds (missing or bad objects), and if you use the diff --git a/fsck-cache.c b/fsck-cache.c index 916792417d5..eae73cc18b0 100644 --- a/fsck-cache.c +++ b/fsck-cache.c @@ -12,6 +12,8 @@ static int show_root = 0; static int show_tags = 0; static int show_unreachable = 0; +static int standalone = 0; +static int check_full = 0; static int keep_cache_objects = 0; static unsigned char head_sha1[20]; @@ -25,13 +27,17 @@ static void check_connectivity(void) struct object_list *refs; if (!obj->parsed) { - printf("missing %s %s\n", - obj->type, sha1_to_hex(obj->sha1)); + if (!standalone && has_sha1_file(obj->sha1)) + ; /* it is in pack */ + else + printf("missing %s %s\n", + obj->type, sha1_to_hex(obj->sha1)); continue; } for (refs = obj->refs; refs; refs = refs->next) { - if (refs->item->parsed) + if (refs->item->parsed || + (!standalone && has_sha1_file(refs->item->sha1))) continue; printf("broken link from %7s %s\n", obj->type, sha1_to_hex(obj->sha1)); @@ -315,8 +321,11 @@ static int read_sha1_reference(const char *path) return -1; obj = lookup_object(sha1); - if (!obj) + if (!obj) { + if (!standalone && has_sha1_file(sha1)) + return 0; /* it is in pack */ return error("%s: invalid sha1 pointer %.40s", path, hexname); + } obj->used = 1; mark_reachable(obj, REACHABLE); @@ -366,10 +375,20 @@ static void get_default_heads(void) die("No default references"); } +static void fsck_object_dir(const char *path) +{ + int i; + for (i = 0; i < 256; i++) { + static char dir[4096]; + sprintf(dir, "%s/%02x", path, i); + fsck_dir(i, dir); + } + fsck_sha1_list(); +} + int main(int argc, char **argv) { int i, heads; - char *sha1_dir; for (i = 1; i < argc; i++) { const char *arg = argv[i]; @@ -390,17 +409,45 @@ int main(int argc, char **argv) keep_cache_objects = 1; continue; } + if (!strcmp(arg, "--standalone")) { + standalone = 1; + continue; + } + if (!strcmp(arg, "--full")) { + check_full = 1; + continue; + } if (*arg == '-') - usage("git-fsck-cache [--tags] [[--unreachable] [--cache] *]"); + usage("git-fsck-cache [--tags] [[--unreachable] [--cache] [--standalone | --full] *]"); } - sha1_dir = get_object_directory(); - for (i = 0; i < 256; i++) { - static char dir[4096]; - sprintf(dir, "%s/%02x", sha1_dir, i); - fsck_dir(i, dir); + if (standalone && check_full) + die("Only one of --standalone or --full can be used."); + if (standalone) + unsetenv("GIT_ALTERNATE_OBJECT_DIRECTORIES"); + + fsck_object_dir(get_object_directory()); + if (check_full) { + int j; + struct packed_git *p; + prepare_alt_odb(); + for (j = 0; alt_odb[j].base; j++) { + alt_odb[j].name[-1] = 0; /* was slash */ + fsck_object_dir(alt_odb[j].base); + alt_odb[j].name[-1] = '/'; + } + prepare_packed_git(); + for (p = packed_git; p; p = p->next) { + int num = num_packed_objects(p); + for (i = 0; i < num; i++) { + unsigned char sha1[20]; + nth_packed_object_sha1(p, i, sha1); + if (fsck_sha1(sha1) < 0) + fprintf(stderr, "bad sha1 entry '%s'\n", sha1_to_hex(sha1)); + + } + } } - fsck_sha1_list(); heads = 0; for (i = 1; i < argc; i++) {