From 2da9ee0888a8570f9a85ef11f208556c5316e3d4 Mon Sep 17 00:00:00 2001 From: Robert Allan Zeh Date: Sun, 19 Jul 2009 18:00:52 -0500 Subject: git svn: add gc command Add a git svn gc command that gzips all unhandled.log files, and removes all index files under .git/svn. Signed-off-by: Robert Allan Zeh Signed-off-by: Eric Wong --- Documentation/git-svn.txt | 4 ++++ git-svn.perl | 33 +++++++++++++++++++++++++++++++++ t/t9143-git-svn-gc.sh | 44 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+) create mode 100755 t/t9143-git-svn-gc.sh diff --git a/Documentation/git-svn.txt b/Documentation/git-svn.txt index 10af599b4..068aa58a5 100644 --- a/Documentation/git-svn.txt +++ b/Documentation/git-svn.txt @@ -338,6 +338,10 @@ Any other arguments are passed directly to 'git log' Shows the Subversion externals. Use -r/--revision to specify a specific revision. +'gc':: + Compress $GIT_DIR/svn//unhandled.log files in .git/svn + and remove $GIT_DIR/svn/index files in .git/svn. + 'reset':: Undoes the effects of 'fetch' back to the specified revision. This allows you to re-'fetch' an SVN revision. Normally the diff --git a/git-svn.perl b/git-svn.perl index 9369acc4d..70159204c 100755 --- a/git-svn.perl +++ b/git-svn.perl @@ -31,6 +31,7 @@ require SVN::Delta; if ($SVN::Core::VERSION lt '1.1.0') { fatal "Need SVN::Core 1.1.0 or better (got $SVN::Core::VERSION)"; } +my $can_compress = eval { require Compress::Zlib; 1}; push @Git::SVN::Ra::ISA, 'SVN::Ra'; push @SVN::Git::Editor::ISA, 'SVN::Delta::Editor'; push @SVN::Git::Fetcher::ISA, 'SVN::Delta::Editor'; @@ -40,6 +41,7 @@ use IO::File qw//; use File::Basename qw/dirname basename/; use File::Path qw/mkpath/; use File::Spec; +use File::Find; use Getopt::Long qw/:config gnu_getopt no_ignore_case auto_abbrev/; use IPC::Open3; use Git; @@ -217,6 +219,10 @@ my %cmd = ( "Undo fetches back to the specified SVN revision", { 'revision|r=s' => \$_revision, 'parent|p' => \$_fetch_parent } ], + 'gc' => [ \&cmd_gc, + "Compress unhandled.log files in .git/svn and remove " . + "index files in .git/svn", + {} ], ); my $cmd; @@ -1107,6 +1113,14 @@ sub cmd_reset { print "r$r = $c ($gs->{ref_id})\n"; } +sub cmd_gc { + if (!$can_compress) { + warn "Compress::Zlib could not be found; unhandled.log " . + "files will not be compressed.\n"; + } + find({ wanted => \&gc_directory, no_chdir => 1}, "$ENV{GIT_DIR}/svn"); +} + ########################### utility functions ######################### sub rebase_cmd { @@ -1527,6 +1541,25 @@ sub md5sum { return $md5->hexdigest(); } +sub gc_directory { + if ($can_compress && -f $_ && basename($_) eq "unhandled.log") { + my $out_filename = $_ . ".gz"; + open my $in_fh, "<", $_ or die "Unable to open $_: $!\n"; + binmode $in_fh; + my $gz = Compress::Zlib::gzopen($out_filename, "ab") or + die "Unable to open $out_filename: $!\n"; + + my $res; + while ($res = sysread($in_fh, my $str, 1024)) { + $gz->gzwrite($str) or + die "Unable to write: ".$gz->gzerror()."!\n"; + } + unlink $_ or die "unlink $File::Find::name: $!\n"; + } elsif (-f $_ && basename($_) eq "index") { + unlink $_ or die "unlink $_: $!\n"; + } +} + package Git::SVN; use strict; use warnings; diff --git a/t/t9143-git-svn-gc.sh b/t/t9143-git-svn-gc.sh new file mode 100755 index 000000000..aaa3af014 --- /dev/null +++ b/t/t9143-git-svn-gc.sh @@ -0,0 +1,44 @@ +#!/bin/sh +# +# Copyright (c) 2009 Robert Allan Zeh + +test_description='git svn gc basic tests' + +. ./lib-git-svn.sh + +test_expect_success 'setup directories and test repo' ' + mkdir import && + mkdir tmp && + echo "Sample text for Subversion repository." > import/test.txt && + svn_cmd import -m "import for git svn" import "$svnrepo" > /dev/null + ' + +test_expect_success 'checkout working copy from svn' \ + 'svn_cmd co "$svnrepo" test_wc' + +test_expect_success 'set some properties to create an unhandled.log file' ' + ( + cd test_wc && + svn_cmd propset foo bar test.txt && + svn_cmd commit -m "property set" + )' + +test_expect_success 'Setup repo' 'git svn init "$svnrepo"' + +test_expect_success 'Fetch repo' 'git svn fetch' + +test_expect_success 'make backup copy of unhandled.log' ' + cp .git/svn/git-svn/unhandled.log tmp + ' + +test_expect_success 'git svn gc runs' 'git svn gc' + +test_expect_success 'git svn gc produces a valid gzip file' ' + gunzip .git/svn/git-svn/unhandled.log.gz + ' + +test_expect_success 'git svn gc does not change unhandled.log files' ' + test_cmp .git/svn/git-svn/unhandled.log tmp/unhandled.log + ' + +test_done -- cgit v1.2.1 From 6b48829dbbe06798eded1e7c5f70810940591f79 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Sat, 25 Jul 2009 00:00:50 -0700 Subject: git svn: revert default behavior for --minimize-url This reverts the --minimize-url behavior change that appeared recently in commit 0b2af457a49e3b00d47d556d5301934d27909db8 ("Fix branch detection when repository root is inaccessible"). However, we now allow the option to be turned off by allowing "--no-minimize-url" so people with limited-access setups can still take advantage of the fix in 0b2af457a49e3b00d47d556d5301934d27909db8. Also document the behavior and default settings of minimize-url in the manpage for the first time. This introduces a temporary UI regression to allow t9141 to pass that will be reverted (fixed) in the next commit. Signed-off-by: Eric Wong --- Documentation/git-svn.txt | 11 +++++++++++ git-svn.perl | 7 ++++++- t/t9141-git-svn-multiple-branches.sh | 8 ++++---- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/Documentation/git-svn.txt b/Documentation/git-svn.txt index 068aa58a5..22a0389f1 100644 --- a/Documentation/git-svn.txt +++ b/Documentation/git-svn.txt @@ -80,6 +80,17 @@ COMMANDS When passed to 'init' or 'clone' this regular expression will be preserved as a config key. See 'fetch' for a description of '--ignore-paths'. +--no-minimize-url;; + When tracking multiple directories (using --stdlayout, + --branches, or --tags options), git svn will attempt to connect + to the root (or highest allowed level) of the Subversion + repository. This default allows better tracking of history if + entire projects are moved within a repository, but may cause + issues on repositories where read access restrictions are in + place. Passing '--no-minimize-url' will allow git svn to + accept URLs as-is without attempting to connect to a higher + level directory. This option is off by default when only + one URL/branch is tracked (it would do little good). 'fetch':: Fetch unfetched revisions from the Subversion remote we are diff --git a/git-svn.perl b/git-svn.perl index 70159204c..10b77ad34 100755 --- a/git-svn.perl +++ b/git-svn.perl @@ -19,6 +19,7 @@ $ENV{GIT_DIR} ||= '.git'; $Git::SVN::default_repo_id = 'svn'; $Git::SVN::default_ref_id = $ENV{GIT_SVN_ID} || 'git-svn'; $Git::SVN::Ra::_log_window_size = 100; +$Git::SVN::_minimize_url = 'unset'; $Git::SVN::Log::TZ = $ENV{TZ}; $ENV{TZ} = 'UTC'; @@ -100,7 +101,7 @@ my %init_opts = ( 'template=s' => \$_template, 'shared:s' => \$_shared, 'trunk|T=s' => \$_trunk, 'tags|t=s@' => \@_tags, 'branches|b=s@' => \@_branches, 'prefix=s' => \$_prefix, 'stdlayout|s' => \$_stdlayout, - 'minimize-url|m' => \$Git::SVN::_minimize_url, + 'minimize-url|m!' => \$Git::SVN::_minimize_url, 'no-metadata' => sub { $icv{noMetadata} = 1 }, 'use-svm-props' => sub { $icv{useSvmProps} = 1 }, 'use-svnsync-props' => sub { $icv{useSvnsyncProps} = 1 }, @@ -399,6 +400,10 @@ sub cmd_init { init_subdir(@_); do_git_init_db(); + if ($Git::SVN::_minimize_url eq 'unset') { + $Git::SVN::_minimize_url = 0; + } + Git::SVN->init($url); } diff --git a/t/t9141-git-svn-multiple-branches.sh b/t/t9141-git-svn-multiple-branches.sh index 3cd06718e..cb9a6d229 100755 --- a/t/t9141-git-svn-multiple-branches.sh +++ b/t/t9141-git-svn-multiple-branches.sh @@ -99,22 +99,22 @@ test_expect_success 'Multiple branch or tag paths require -d' ' test_expect_success 'create new branches and tags' ' ( cd git_project && - git svn branch -m "New branch 1" -d b_one New1 ) && + git svn branch -m "New branch 1" -d project/b_one New1 ) && ( cd svn_project && svn_cmd up && test -e b_one/New1/a.file ) && ( cd git_project && - git svn branch -m "New branch 2" -d b_two New2 ) && + git svn branch -m "New branch 2" -d project/b_two New2 ) && ( cd svn_project && svn_cmd up && test -e b_two/New2/a.file ) && ( cd git_project && - git svn branch -t -m "New tag 1" -d tags_A Tag1 ) && + git svn branch -t -m "New tag 1" -d project/tags_A Tag1 ) && ( cd svn_project && svn_cmd up && test -e tags_A/Tag1/a.file ) && ( cd git_project && - git svn tag -m "New tag 2" -d tags_B Tag2 ) && + git svn tag -m "New tag 2" -d project/tags_B Tag2 ) && ( cd svn_project && svn_cmd up && test -e tags_B/Tag2/a.file ) ' -- cgit v1.2.1 From eaa14ff8c7dbe51dfdfe7e2d9ede839fc62dcf52 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Sat, 25 Jul 2009 01:36:06 -0700 Subject: git svn: the branch command no longer needs the full path This was introduced in 0b2af457a49e3b00d47d556d5301934d27909db8 ("Fix branch detection when repository root is inaccessible") but reintroduced in the previous commit. Signed-off-by: Eric Wong --- git-svn.perl | 19 ++++++++++++++++--- t/t9141-git-svn-multiple-branches.sh | 8 ++++---- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/git-svn.perl b/git-svn.perl index 10b77ad34..9808597a1 100755 --- a/git-svn.perl +++ b/git-svn.perl @@ -666,9 +666,22 @@ sub cmd_branch { } } unless (defined $glob) { - die "Unknown ", - $_tag ? "tag" : "branch", - " destination $_branch_dest\n"; + my $dest_re = qr/\b\Q$_branch_dest\E\b/; + foreach my $g (@{$allglobs}) { + $g->{path}->{left} =~ /$dest_re/ or next; + if (defined $glob) { + die "Ambiguous destination: ", + $_branch_dest, "\nmatches both '", + $glob->{path}->{left}, "' and '", + $g->{path}->{left}, "'\n"; + } + $glob = $g; + } + unless (defined $glob) { + die "Unknown ", + $_tag ? "tag" : "branch", + " destination $_branch_dest\n"; + } } } my ($lft, $rgt) = @{ $glob->{path} }{qw/left right/}; diff --git a/t/t9141-git-svn-multiple-branches.sh b/t/t9141-git-svn-multiple-branches.sh index cb9a6d229..3cd06718e 100755 --- a/t/t9141-git-svn-multiple-branches.sh +++ b/t/t9141-git-svn-multiple-branches.sh @@ -99,22 +99,22 @@ test_expect_success 'Multiple branch or tag paths require -d' ' test_expect_success 'create new branches and tags' ' ( cd git_project && - git svn branch -m "New branch 1" -d project/b_one New1 ) && + git svn branch -m "New branch 1" -d b_one New1 ) && ( cd svn_project && svn_cmd up && test -e b_one/New1/a.file ) && ( cd git_project && - git svn branch -m "New branch 2" -d project/b_two New2 ) && + git svn branch -m "New branch 2" -d b_two New2 ) && ( cd svn_project && svn_cmd up && test -e b_two/New2/a.file ) && ( cd git_project && - git svn branch -t -m "New tag 1" -d project/tags_A Tag1 ) && + git svn branch -t -m "New tag 1" -d tags_A Tag1 ) && ( cd svn_project && svn_cmd up && test -e tags_A/Tag1/a.file ) && ( cd git_project && - git svn tag -m "New tag 2" -d project/tags_B Tag2 ) && + git svn tag -m "New tag 2" -d tags_B Tag2 ) && ( cd svn_project && svn_cmd up && test -e tags_B/Tag2/a.file ) ' -- cgit v1.2.1 From 5af9b77b553eae80a14dd06f14bf1ee1268a0288 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Sat, 25 Jul 2009 02:11:39 -0700 Subject: t9142: stop httpd after the test Otherwise it would fail in subsequent runs if the same SVN_HTTPD_PORT was used. Signed-off-by: Eric Wong --- t/t9142-git-svn-shallow-clone.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/t/t9142-git-svn-shallow-clone.sh b/t/t9142-git-svn-shallow-clone.sh index fd5ad4947..1236accd9 100755 --- a/t/t9142-git-svn-shallow-clone.sh +++ b/t/t9142-git-svn-shallow-clone.sh @@ -27,4 +27,6 @@ test_expect_success 'clone trunk with "-r HEAD"' ' ( cd g && git rev-parse --symbolic --verify HEAD ) ' +stop_httpd + test_done -- cgit v1.2.1 From 884cce5bd042e67a0d2a1a9317f8435634486ad1 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Sat, 25 Jul 2009 02:29:28 -0700 Subject: git svn: avoid escaping '/' when renaming/copying files Timothy Schaeffer reported the following: > Git-svn has been giving me the following error for some time > when calling "git svn dcommit": > > RA layer request failed: PROPFIND request failed on > '/svn/stf/branches/dev/sw%2Fdpemu%2Finclude%2FNetCnxn.h': PROPFIND of > '/svn/stf/branches/dev/sw%2Fdpemu%2Finclude%2FNetCnxn.h': 302 Found > (https://oursvnrepo.net) at /usr/local/libexec/git-core/git-svn line 508 > > This only occurred when git detected a rename or copy. > > Following the lead into git-svn.perl, > and noticing that some of the '/'s in the path were hex-encoded > and some were not, > I changed the regex used to find chars > to hex-encode in the relative part of the path > to exclude '/'. > It works, so far. > I have included a patch. While this has previous not been a problem in my experience, newer versions of SVN may be stricter and this does not introduce regressions in t9115. Signed-off-by: Eric Wong --- git-svn.perl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git-svn.perl b/git-svn.perl index 9808597a1..fd7232cf6 100755 --- a/git-svn.perl +++ b/git-svn.perl @@ -4005,7 +4005,7 @@ sub repo_path { sub url_path { my ($self, $path) = @_; if ($self->{url} =~ m#^https?://#) { - $path =~ s/([^~a-zA-Z0-9_.-])/uc sprintf("%%%02x",ord($1))/eg; + $path =~ s!([^~a-zA-Z0-9_./-])!uc sprintf("%%%02x",ord($1))!eg; } $self->{url} . '/' . $self->repo_path($path); } -- cgit v1.2.1 From 5f8b2cbacd3bad38d0ae4d358fe9b34e98190935 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Sat, 25 Jul 2009 13:14:16 -0700 Subject: git svn: make minimize URL more reliable over http(s) In addition to path-based restrictions, Subversion servers over http(s) may have access controls implemented via the LimitExcept directive in Apache. In some cases, LimitExcept may be (arguably) misconfigured to not allow REPORT requests while allowing OPTIONS and PROPFIND. This caused problems with our existing minimize_url logic that only issued OPTIONS and PROPFIND requests when connecting and using SVN::Ra::get_latest_revnum. We now call SVN::Ra::get_log if get_latest_revnum succeeds, resulting in a REPORT request being sent. This will increase our chances of tripping access controls before we start attempting to fetch history. Signed-off-by: Eric Wong --- git-svn.perl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/git-svn.perl b/git-svn.perl index fd7232cf6..d07581072 100755 --- a/git-svn.perl +++ b/git-svn.perl @@ -4831,7 +4831,11 @@ sub minimize_url { my $c = ''; do { $url .= "/$c" if length $c; - eval { (ref $self)->new($url)->get_latest_revnum }; + eval { + my $ra = (ref $self)->new($url); + my $latest = $ra->get_latest_revnum; + $ra->get_log("", $latest, 0, 1, 0, 1, sub {}); + }; } while ($@ && ($c = shift @components)); $url; } -- cgit v1.2.1