diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | Makefile | 11 | ||||
| -rw-r--r-- | builtin/clone.c | 16 | ||||
| -rw-r--r-- | fast-import.c | 2 | ||||
| -rw-r--r-- | git-remote-testgit.py | 235 | ||||
| -rw-r--r-- | git_remote_helpers/git/exporter.py | 51 | ||||
| -rw-r--r-- | git_remote_helpers/git/importer.py | 38 | ||||
| -rw-r--r-- | git_remote_helpers/git/non_local.py | 61 | ||||
| -rw-r--r-- | git_remote_helpers/git/repo.py | 70 | ||||
| -rwxr-xr-x | t/t5800-remote-helpers.sh | 76 | ||||
| -rw-r--r-- | transport-helper.c | 137 |
11 files changed, 671 insertions, 27 deletions
diff --git a/.gitignore b/.gitignore index dbf1b90c6..14e2b6bde 100644 --- a/.gitignore +++ b/.gitignore @@ -112,6 +112,7 @@ /git-remote-https /git-remote-ftp /git-remote-ftps +/git-remote-testgit /git-repack /git-replace /git-repo-config @@ -366,6 +366,8 @@ SCRIPT_PERL += git-relink.perl SCRIPT_PERL += git-send-email.perl SCRIPT_PERL += git-svn.perl +SCRIPT_PYTHON += git-remote-testgit.py + SCRIPTS = $(patsubst %.sh,%,$(SCRIPT_SH)) \ $(patsubst %.perl,%,$(SCRIPT_PERL)) \ $(patsubst %.py,%,$(SCRIPT_PYTHON)) \ @@ -1622,13 +1624,8 @@ $(patsubst %.py,%,$(SCRIPT_PYTHON)): % : %.py INSTLIBDIR=`MAKEFLAGS= $(MAKE) -C git_remote_helpers -s \ --no-print-directory prefix='$(prefix_SQ)' DESTDIR='$(DESTDIR_SQ)' \ instlibdir` && \ - sed -e '1{' \ - -e ' s|#!.*python|#!$(PYTHON_PATH_SQ)|' \ - -e '}' \ - -e 's|^import sys.*|&; \\\ - import os; \\\ - sys.path.insert(0, os.getenv("GITPYTHONLIB",\ - "@@INSTLIBDIR@@"));|' \ + sed -e '1s|#!.*python|#!$(PYTHON_PATH_SQ)|' \ + -e 's|\(os\.getenv("GITPYTHONLIB"\)[^)]*)|\1,"@@INSTLIBDIR@@")|' \ -e 's|@@INSTLIBDIR@@|'"$$INSTLIBDIR"'|g' \ $@.py >$@+ && \ chmod +x $@+ && \ diff --git a/builtin/clone.c b/builtin/clone.c index 0bedde41f..445792242 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -474,9 +474,6 @@ int cmd_clone(int argc, const char **argv, const char *prefix) */ unsetenv(CONFIG_ENVIRONMENT); - if (option_reference) - setup_reference(git_dir); - git_config(git_default_config, NULL); if (option_bare) { @@ -502,12 +499,15 @@ int cmd_clone(int argc, const char **argv, const char *prefix) git_config_set(key.buf, "true"); strbuf_reset(&key); } - - strbuf_addf(&key, "remote.%s.url", option_origin); - git_config_set(key.buf, repo); - strbuf_reset(&key); } + strbuf_addf(&key, "remote.%s.url", option_origin); + git_config_set(key.buf, repo); + strbuf_reset(&key); + + if (option_reference) + setup_reference(git_dir); + fetch_pattern = value.buf; refspec = parse_fetch_refspec(1, &fetch_pattern); @@ -517,7 +517,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix) refs = clone_local(path, git_dir); mapped_refs = wanted_peer_refs(refs, refspec); } else { - struct remote *remote = remote_get(argv[0]); + struct remote *remote = remote_get(option_origin); transport = transport_get(remote, remote->url[0]); if (!transport->get_refs_list || !transport->fetch) diff --git a/fast-import.c b/fast-import.c index 309f2c58a..129a78683 100644 --- a/fast-import.c +++ b/fast-import.c @@ -2707,6 +2707,7 @@ static void option_import_marks(const char *marks, int from_stream) } import_marks_file = make_fast_import_path(marks); + safe_create_leading_directories_const(import_marks_file); import_marks_file_from_stream = from_stream; } @@ -2737,6 +2738,7 @@ static void option_active_branches(const char *branches) static void option_export_marks(const char *marks) { export_marks_file = make_fast_import_path(marks); + safe_create_leading_directories_const(export_marks_file); } static void option_export_pack_edges(const char *edges) diff --git a/git-remote-testgit.py b/git-remote-testgit.py new file mode 100644 index 000000000..92539222c --- /dev/null +++ b/git-remote-testgit.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python + +import hashlib +import sys +import os +sys.path.insert(0, os.getenv("GITPYTHONLIB",".")) + +from git_remote_helpers.util import die, debug, warn +from git_remote_helpers.git.repo import GitRepo +from git_remote_helpers.git.exporter import GitExporter +from git_remote_helpers.git.importer import GitImporter +from git_remote_helpers.git.non_local import NonLocalGit + +def get_repo(alias, url): + """Returns a git repository object initialized for usage. + """ + + repo = GitRepo(url) + repo.get_revs() + repo.get_head() + + hasher = hashlib.sha1() + hasher.update(repo.path) + repo.hash = hasher.hexdigest() + + repo.get_base_path = lambda base: os.path.join( + base, 'info', 'fast-import', repo.hash) + + prefix = 'refs/testgit/%s/' % alias + debug("prefix: '%s'", prefix) + + repo.gitdir = "" + repo.alias = alias + repo.prefix = prefix + + repo.exporter = GitExporter(repo) + repo.importer = GitImporter(repo) + repo.non_local = NonLocalGit(repo) + + return repo + + +def local_repo(repo, path): + """Returns a git repository object initalized for usage. + """ + + local = GitRepo(path) + + local.non_local = None + local.gitdir = repo.gitdir + local.alias = repo.alias + local.prefix = repo.prefix + local.hash = repo.hash + local.get_base_path = repo.get_base_path + local.exporter = GitExporter(local) + local.importer = GitImporter(local) + + return local + + +def do_capabilities(repo, args): + """Prints the supported capabilities. + """ + + print "import" + print "export" + print "gitdir" + print "refspec refs/heads/*:%s*" % repo.prefix + + print # end capabilities + + +def do_list(repo, args): + """Lists all known references. + + Bug: This will always set the remote head to master for non-local + repositories, since we have no way of determining what the remote + head is at clone time. + """ + + for ref in repo.revs: + debug("? refs/heads/%s", ref) + print "? refs/heads/%s" % ref + + if repo.head: + debug("@refs/heads/%s HEAD" % repo.head) + print "@refs/heads/%s HEAD" % repo.head + else: + debug("@refs/heads/master HEAD") + print "@refs/heads/master HEAD" + + print # end list + + +def update_local_repo(repo): + """Updates (or clones) a local repo. + """ + + if repo.local: + return repo + + path = repo.non_local.clone(repo.gitdir) + repo.non_local.update(repo.gitdir) + repo = local_repo(repo, path) + return repo + + +def do_import(repo, args): + """Exports a fast-import stream from testgit for git to import. + """ + + if len(args) != 1: + die("Import needs exactly one ref") + + if not repo.gitdir: + die("Need gitdir to import") + + repo = update_local_repo(repo) + repo.exporter.export_repo(repo.gitdir) + + +def do_export(repo, args): + """Imports a fast-import stream from git to testgit. + """ + + if not repo.gitdir: + die("Need gitdir to export") + + dirname = repo.get_base_path(repo.gitdir) + + if not os.path.exists(dirname): + os.makedirs(dirname) + + path = os.path.join(dirname, 'testgit.marks') + print path + print path if os.path.exists(path) else "" + sys.stdout.flush() + + update_local_repo(repo) + repo.importer.do_import(repo.gitdir) + repo.non_local.push(repo.gitdir) + + +def do_gitdir(repo, args): + """Stores the location of the gitdir. + """ + + if not args: + die("gitdir needs an argument") + + repo.gitdir = ' '.join(args) + + +COMMANDS = { + 'capabilities': do_capabilities, + 'list': do_list, + 'import': do_import, + 'export': do_export, + 'gitdir': do_gitdir, +} + + +def sanitize(value): + """Cleans up the url. + """ + + if value.startswith('testgit::'): + value = value[9:] + + return value + + +def read_one_line(repo): + """Reads and processes one command. + """ + + line = sys.stdin.readline() + + cmdline = line + + if not cmdline: + warn("Unexpected EOF") + return False + + cmdline = cmdline.strip().split() + if not cmdline: + # Blank line means we're about to quit + return False + + cmd = cmdline.pop(0) + debug("Got command '%s' with args '%s'", cmd, ' '.join(cmdline)) + + if cmd not in COMMANDS: + die("Unknown command, %s", cmd) + + func = COMMANDS[cmd] + func(repo, cmdline) + sys.stdout.flush() + + return True + + +def main(args): + """Starts a new remote helper for the specified repository. + """ + + if len(args) != 3: + die("Expecting exactly three arguments.") + sys.exit(1) + + if os.getenv("GIT_DEBUG_TESTGIT"): + import git_remote_helpers.util + git_remote_helpers.util.DEBUG = True + + alias = sanitize(args[1]) + url = sanitize(args[2]) + + if not alias.isalnum(): + warn("non-alnum alias '%s'", alias) + alias = "tmp" + + args[1] = alias + args[2] = url + + repo = get_repo(alias, url) + + debug("Got arguments %s", args[1:]) + + more = True + + while (more): + more = read_one_line(repo) + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/git_remote_helpers/git/exporter.py b/git_remote_helpers/git/exporter.py new file mode 100644 index 000000000..dfaab00b5 --- /dev/null +++ b/git_remote_helpers/git/exporter.py @@ -0,0 +1,51 @@ +import os +import subprocess +import sys + + +class GitExporter(object): + """An exporter for testgit repositories. + + The exporter simply delegates to git fast-export. + """ + + def __init__(self, repo): + """Creates a new exporter for the specified repo. + """ + + self.repo = repo + + def export_repo(self, base): + """Exports a fast-export stream for the given directory. + + Simply delegates to git fast-epxort and pipes it through sed + to make the refs show up under the prefix rather than the + default refs/heads. This is to demonstrate how the export + data can be stored under it's own ref (using the refspec + capability). + """ + + dirname = self.repo.get_base_path(base) + path = os.path.abspath(os.path.join(dirname, 'testgit.marks')) + + if not os.path.exists(dirname): + os.makedirs(dirname) + + print "feature relative-marks" + if os.path.exists(os.path.join(dirname, 'git.marks')): + print "feature import-marks=%s/git.marks" % self.repo.hash + print "feature export-marks=%s/git.marks" % self.repo.hash + sys.stdout.flush() + + args = ["git", "--git-dir=" + self.repo.gitpath, "fast-export", "--export-marks=" + path] + + if os.path.exists(path): + args.append("--import-marks=" + path) + + args.append("HEAD") + + p1 = subprocess.Popen(args, stdout=subprocess.PIPE) + + args = ["sed", "s_refs/heads/_" + self.repo.prefix + "_g"] + + subprocess.check_call(args, stdin=p1.stdout) diff --git a/git_remote_helpers/git/importer.py b/git_remote_helpers/git/importer.py new file mode 100644 index 000000000..af2919d92 --- /dev/null +++ b/git_remote_helpers/git/importer.py @@ -0,0 +1,38 @@ +import os +import subprocess + + +class GitImporter(object): + """An importer for testgit repositories. + + This importer simply delegates to git fast-import. + """ + + def __init__(self, repo): + """Creates a new importer for the specified repo. + """ + + self.repo = repo + + def do_import(self, base): + """Imports a fast-import stream to the given directory. + + Simply delegates to git fast-import. + """ + + dirname = self.repo.get_base_path(base) + if self.repo.local: + gitdir = self.repo.gitpath + else: + gitdir = os.path.abspath(os.path.join(dirname, '.git')) + path = os.path.abspath(os.path.join(dirname, 'git.marks')) + + if not os.path.exists(dirname): + os.makedirs(dirname) + + args = ["git", "--git-dir=" + gitdir, "fast-import", "--quiet", "--export-marks=" + path] + + if os.path.exists(path): + args.append("--import-marks=" + path) + + subprocess.check_call(args) diff --git a/git_remote_helpers/git/non_local.py b/git_remote_helpers/git/non_local.py new file mode 100644 index 000000000..d75ef8f21 --- /dev/null +++ b/git_remote_helpers/git/non_local.py @@ -0,0 +1,61 @@ +import os +import subprocess + +from git_remote_helpers.util import die, warn + + +class NonLocalGit(object): + """Handler to interact with non-local repos. + """ + + def __init__(self, repo): + """Creates a new non-local handler for the specified repo. + """ + + self.repo = repo + + def clone(self, base): + """Clones the non-local repo to base. + + Does nothing if a clone already exists. + """ + + path = os.path.join(self.repo.get_base_path(base), '.git') + + # already cloned + if os.path.exists(path): + return path + + os.makedirs(path) + args = ["git", "clone", "--bare", "--quiet", self.repo.gitpath, path] + + subprocess.check_call(args) + + return path + + def update(self, base): + """Updates checkout of the non-local repo in base. + """ + + path = os.path.join(self.repo.get_base_path(base), '.git') + + if not os.path.exists(path): + die("could not find repo at %s", path) + + args = ["git", "--git-dir=" + path, "fetch", "--quiet", self.repo.gitpath] + subprocess.check_call(args) + + args = ["git", "--git-dir=" + path, "update-ref", "refs/heads/master", "FETCH_HEAD"] + subprocess.check_call(args) + + def push(self, base): + """Pushes from the non-local repo to base. + """ + + path = os.path.join(self.repo.get_base_path(base), '.git') + + if not os.path.exists(path): + die("could not find repo at %s", path) + + args = ["git", "--git-dir=" + path, "push", "--quiet", self.repo.gitpath] + subprocess.check_call(args) diff --git a/git_remote_helpers/git/repo.py b/git_remote_helpers/git/repo.py new file mode 100644 index 000000000..82d5f78c7 --- /dev/null +++ b/git_remote_helpers/git/repo.py @@ -0,0 +1,70 @@ +import os +import subprocess + +def sanitize(rev, sep='\t'): + """Converts a for-each-ref line to a name/value pair. + """ + + splitrev = rev.split(sep) + branchval = splitrev[0] + branchname = splitrev[1].strip() + if branchname.startswith("refs/heads/"): + branchname = branchname[11:] + + return branchname, branchval + +def is_remote(url): + """Checks whether the specified value is a remote url. + """ + + prefixes = ["http", "file", "git"] + + return any(url.startswith(i) for i in prefixes) + +class GitRepo(object): + """Repo object representing a repo. + """ + + def __init__(self, path): + """Initializes a new repo at the given path. + """ + + self.path = path + self.head = None + self.revmap = {} + self.local = not is_remote(self.path) + + if(self.path.endswith('.git')): + self.gitpath = self.path + else: + self.gitpath = os.path.join(self.path, '.git') + + if self.local and not os.path.exists(self.gitpath): + os.makedirs(self.gitpath) + + def get_revs(self): + """Fetches all revs from the remote. + """ + + args = ["git", "ls-remote", self.gitpath] + path = ".cached_revs" + ofile = open(path, "w") + + subprocess.check_call(args, stdout=ofile) + output = open(path).readlines() + self.revmap = dict(sanitize(i) for i in output) + if "HEAD" in self.revmap: + del self.revmap["HEAD"] + self.revs = self.revmap.keys() + ofile.close() + + def get_head(self): + """Determines the head of a local repo. + """ + + if not self.local: + return + + path = os.path.join(self.gitpath, "HEAD") + head = open(path).readline() + self.head, _ = sanitize(head, ' ') diff --git a/t/t5800-remote-helpers.sh b/t/t5800-remote-helpers.sh new file mode 100755 index 000000000..75a0163c0 --- /dev/null +++ b/t/t5800-remote-helpers.sh @@ -0,0 +1,76 @@ +#!/bin/sh +# +# Copyright (c) 2010 Sverre Rabbelier +# + +test_description='Test remote-helper import and export commands' + +. ./test-lib.sh + +if ! test_have_prereq PYTHON +then + say 'skipping git remote-testgit tests: requires Python support' + test_done +fi + +test_expect_success 'setup repository' ' + git init --bare server/.git && + git clone server public && + (cd public && + echo content >file && + git add file && + git commit -m one && + git push origin master) +' + +test_expect_success 'cloning from local repo' ' + git clone "testgit::${PWD}/server" localclone && + test_cmp public/file localclone/file +' + +test_expect_success 'cloning from remote repo' ' + git clone "testgit::file://${PWD}/server" clone && + test_cmp public/file clone/file +' + +test_expect_success 'create new commit on remote' ' + (cd public && + echo content >>file && + git commit -a -m two && + git push) +' + +test_expect_success 'pulling from local repo' ' + (cd localclone && git pull) && + test_cmp public/file localclone/file +' + +test_expect_success 'pulling from remote remote' ' + (cd clone && git pull) && + test_cmp public/file clone/file +' + +test_expect_success 'pushing to local repo' ' + (cd localclone && + echo content >>file && + git commit -a -m three && + git push) && + HEAD=$(git --git-dir=localclone/.git rev-parse --verify HEAD) && + test $HEAD = $(git --git-dir=server/.git rev-parse --verify HEAD) +' + +test_expect_success 'synch with changes from localclone' ' + (cd clone && + git pull) +' + +test_expect_success 'pushing remote local repo' ' + (cd clone && + echo content >>file && + git commit -a -m four && + git push) && + HEAD=$(git --git-dir=clone/.git rev-parse --verify HEAD) && + test $HEAD = $(git --git-dir=server/.git rev-parse --verify HEAD) +' + +test_done diff --git a/transport-helper.c b/transport-helper.c index 2638781c5..0381de536 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -7,6 +7,7 @@ #include "revision.h" #include "quote.h" #include "remote.h" +#include "string-list.h" static int debug; @@ -17,6 +18,7 @@ struct helper_data FILE *out; unsigned fetch : 1, import : 1, + export : 1, option : 1, push : 1, connect : 1, @@ -163,6 +165,8 @@ static struct child_process *get_helper(struct transport *transport) data->push = 1; else if (!strcmp(capname, "import")) data->import = 1; + else if (!strcmp(capname, "export")) + data->export = 1; else if (!data->refspecs && !prefixcmp(capname, "refspec ")) { ALLOC_GROW(refspecs, refspec_nr + 1, @@ -170,6 +174,11 @@ static struct child_process *get_helper(struct transport *transport) refspecs[refspec_nr++] = strdup(buf.buf + strlen("refspec ")); } else if (!strcmp(capname, "connect")) { data->connect = 1; + } else if (!strcmp(buf.buf, "gitdir")) { + struct strbuf gitdir = STRBUF_INIT; + strbuf_addf(&gitdir, "gitdir %s\n", get_git_dir()); + sendline(data, &gitdir); + strbuf_release(&gitdir); } else if (mandatory) { die("Unknown mandatory capability %s. This remote " "helper probably needs newer version of Git.\n", @@ -351,6 +360,33 @@ static int get_importer(struct transport *transport, struct child_process *fasti return start_command(fastimport); } +static int get_exporter(struct transport *transport, + struct child_process *fastexport, + const char *export_marks, + const char *import_marks, + struct string_list *revlist_args) +{ + struct child_process *helper = get_helper(transport); + int argc = 0, i; + memset(fastexport, 0, sizeof(*fastexport)); + + /* we need to duplicate helper->in because we want to use it after + * fastexport is done with it. */ + fastexport->out = dup(helper->in); + fastexport->argv = xcalloc(4 + revlist_args->nr, sizeof(*fastexport->argv)); + fastexport->argv[argc++] = "fast-export"; + if (export_marks) + fastexport->argv[argc++] = export_marks; + if (import_marks) + fastexport->argv[argc++] = import_marks; + + for (i = 0; i < revlist_args->nr; i++) + fastexport->argv[argc++] = revlist_args->items[i].string; + + fastexport->git_cmd = 1; + return start_command(fastexport); +} + static int fetch_with_import(struct transport *transport, int nr_heads, struct ref **to_fetch) { @@ -518,7 +554,7 @@ static int fetch(struct transport *transport, return -1; } -static int push_refs(struct transport *transport, +static int push_refs_with_push(struct transport *transport, struct ref *remote_refs, int flags) { int force_all = flags & TRANSPORT_PUSH_FORCE; @@ -528,17 +564,6 @@ static int push_refs(struct transport *transport, struct child_process *helper; struct ref *ref; - if (process_connect(transport, 1)) { - do_take_over(transport); - return transport->push_refs(transport, remote_refs, flags); - } - - if (!remote_refs) { - fprintf(stderr, "No refs in common and none specified; doing nothing.\n" - "Perhaps you should specify a branch such as 'master'.\n"); - return 0; - } - helper = get_helper(transport); if (!data->push) return 1; @@ -657,6 +682,94 @@ static int push_refs(struct transport *transport, return 0; } +static int push_refs_with_export(struct transport *transport, + struct ref *remote_refs, int flags) +{ + struct ref *ref; + struct child_process *helper, exporter; + struct helper_data *data = transport->data; + char *export_marks = NULL, *import_marks = NULL; + struct string_list revlist_args = { NULL, 0, 0 }; + struct strbuf buf = STRBUF_INIT; + + helper = get_helper(transport); + + write_constant(helper->in, "export\n"); + + recvline(data, &buf); + if (debug) + fprintf(stderr, "Debug: Got export_marks '%s'\n", buf.buf); + if (buf.len) { + struct strbuf arg = STRBUF_INIT; + strbuf_addstr(&arg, "--export-marks="); + strbuf_addbuf(&arg, &buf); + export_marks = strbuf_detach(&arg, NULL); + } + + recvline(data, &buf); + if (debug) + fprintf(stderr, "Debug: Got import_marks '%s'\n", buf.buf); + if (buf.len) { + struct strbuf arg = STRBUF_INIT; + strbuf_addstr(&arg, "--import-marks="); + strbuf_addbuf(&arg, &buf); + import_marks = strbuf_detach(&arg, NULL); + } + + strbuf_reset(&buf); + + for (ref = remote_refs; ref; ref = ref->next) { + char *private; + unsigned char sha1[20]; + + if (!data->refspecs) + continue; + private = apply_refspecs(data->refspecs, data->refspec_nr, ref->name); + if (private && !get_sha1(private, sha1)) { + strbuf_addf(&buf, "^%s", private); + string_list_append(strbuf_detach(&buf, NULL), &revlist_args); + } + + string_list_append(ref->name, &revlist_args); + + } + + if (get_exporter(transport, &exporter, + export_marks, import_marks, &revlist_args)) + die("Couldn't run fast-export"); + + data->no_disconnect_req = 1; + finish_command(&exporter); + disconnect_helper(transport); + return 0; +} + +static int push_refs(struct transport *transport, + struct ref *remote_refs, int flags) +{ + struct helper_data *data = transport->data; + + if (process_connect(transport, 1)) { + do_take_over(transport); + return transport->push_refs(transport, remote_refs, flags); + } + + if (!remote_refs) { + fprintf(stderr, "No refs in common and none specified; doing nothing.\n" + "Perhaps you should specify a branch such as 'master'.\n"); + return 0; + } + + if (data->push) + return push_refs_with_push(transport, remote_refs, flags); + + if (data->export) + return push_refs_with_export(transport, remote_refs, flags); + + return -1; +} + + static int has_attribute(const char *attrs, const char *attr) { int len; if (!attrs) |