From 563184dc19bb6a695f761ac9001cc67009400b5f Mon Sep 17 00:00:00 2001 From: moregeek Date: Mon, 10 Dec 2012 16:00:54 +0000 Subject: [PATCH 1/4] fixes repo backup problems with empty repos - #1793 & #1793 --- lib/tasks/gitlab/backup.rake | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index c01fe479dba..00f32996159 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -113,17 +113,27 @@ namespace :gitlab do task :repo_dump => :environment do backup_path_repo = File.join(Gitlab.config.backup_path, "repositories") FileUtils.mkdir_p(backup_path_repo) until Dir.exists?(backup_path_repo) + # Gitlab repositories puts "Dumping repositories:" - project = Project.all.map { |n| [n.path, n.path_to_repo] } - project << ["gitolite-admin.git", File.join(File.dirname(project.first.second), "gitolite-admin.git")] - project.each do |project| - print "- Dumping repository #{project.first}... " - if Kernel.system("cd #{project.second} > /dev/null 2>&1 && git bundle create #{backup_path_repo}/#{project.first}.bundle --all > /dev/null 2>&1") - puts "[DONE]".green + Project.all.each do |project| + print "- Dumping repository #{project.name}... " + unless project.empty_repo? + if Kernel.system("cd #{project.path_to_repo} > /dev/null 2>&1 && git bundle create #{backup_path_repo}/#{project.path}.bundle --all > /dev/null 2>&1") + puts "[DONE]".green + else + puts "[FAILED]".red + end else - puts "[FAILED]".red + puts "[SKIPPING]".yellow end end + # Gitolite repositories + print "- Dumping repository gitolite-admin... " + if Kernel.system("cd #{File.join(Gitlab.config.git_base_path, "gitolite-admin.git")} > /dev/null 2>&1 && git bundle create #{backup_path_repo}/gitolite-admin.bundle --all > /dev/null 2>&1") + puts "[DONE]".green + else + puts "[FAILED]".red + end end task :repo_restore => :environment do -- GitLab From 434003564a158477353507c8f2d7fae1acfcd06f Mon Sep 17 00:00:00 2001 From: moregeek Date: Thu, 13 Dec 2012 17:47:45 +0000 Subject: [PATCH 2/4] add method to check if project has a namespace --- app/models/project.rb | 4 ++++ spec/models/project_spec.rb | 14 ++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/app/models/project.rb b/app/models/project.rb index 372b94d2066..46728402f3b 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -251,6 +251,10 @@ class Project < ActiveRecord::Base end end + def namespace? + ! namespace.nil? + end + # For compatibility with old code def code path diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index db0d30727b4..1f4a57c102d 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -86,6 +86,7 @@ describe Project do it { should respond_to(:path_to_repo) } it { should respond_to(:valid_repo?) } it { should respond_to(:repo_exists?) } + it { should respond_to(:namespace?) } # Repository Role it { should respond_to(:tree) } @@ -165,6 +166,19 @@ describe Project do end end + describe :namespace? do + let(:project) { create(:project) } + + it "should return true" do + project.namespace = Namespace.new(path: "some_namespace") + project.namespace?.should be_true + end + + it "should return false" do + project.namespace?.should be_false + end + end + describe "last_activity methods" do let(:project) { create(:project) } let(:last_event) { double(created_at: Time.now) } -- GitLab From 17e07bbdb6ba283be0904873d45ae0c4c20dfc63 Mon Sep 17 00:00:00 2001 From: moregeek Date: Wed, 19 Dec 2012 21:47:19 +0000 Subject: [PATCH 3/4] improved repo dump/restore --- lib/tasks/gitlab/backup.rake | 89 ++++++++++++++++++++++++++++-------- 1 file changed, 69 insertions(+), 20 deletions(-) diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 00f32996159..71227630679 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -113,48 +113,97 @@ namespace :gitlab do task :repo_dump => :environment do backup_path_repo = File.join(Gitlab.config.backup_path, "repositories") FileUtils.mkdir_p(backup_path_repo) until Dir.exists?(backup_path_repo) + # Gitlab repositories puts "Dumping repositories:" Project.all.each do |project| - print "- Dumping repository #{project.name}... " + print "- Dumping repository '#{project.path_with_namespace}.git'... " unless project.empty_repo? - if Kernel.system("cd #{project.path_to_repo} > /dev/null 2>&1 && git bundle create #{backup_path_repo}/#{project.path}.bundle --all > /dev/null 2>&1") + + namespace = project.namespace? ? project.namespace.path : "/" + path_to_bundle = File.join(backup_path_repo, project.path_with_namespace + ".bundle") + FileUtils.mkdir_p(File.join(backup_path_repo, namespace)) + + if Kernel.system("cd #{project.path_to_repo} > /dev/null 2>&1 && git bundle create #{path_to_bundle} --all > /dev/null 2>&1") puts "[DONE]".green else puts "[FAILED]".red end + else puts "[SKIPPING]".yellow end end - # Gitolite repositories - print "- Dumping repository gitolite-admin... " - if Kernel.system("cd #{File.join(Gitlab.config.git_base_path, "gitolite-admin.git")} > /dev/null 2>&1 && git bundle create #{backup_path_repo}/gitolite-admin.bundle --all > /dev/null 2>&1") - puts "[DONE]".green - else - puts "[FAILED]".red + + # Non gitlab managed repositories + [ "gitolite-admin.git" ].each do |repository| + path_to_repo = File.join(Gitlab.config.git_base_path, repository) + path_to_bundle = File.join(backup_path_repo, repository + ".bundle") + + print "- Dumping repository '#{repository}'... " + if Kernel.system("cd #{path_to_repo} > /dev/null 2>&1 && git bundle create #{path_to_bundle} --all > /dev/null 2>&1") + puts "[DONE]".green + else + puts "[FAILED]".red + end end end task :repo_restore => :environment do - backup_path_repo = File.join(Gitlab.config.backup_path, "repositories") + + backup_path_repo = File.join(Gitlab.config.backup_path, "repositories") + repos_in_backup = Dir.chdir(backup_path_repo) && Dir.glob("{*.bundle,*/*.bundle}") + permission_commands = [ + "sudo chmod -R g+rwX #{Gitlab.config.git_base_path}", + "sudo chown -R #{Gitlab.config.ssh_user}:#{Gitlab.config.ssh_user} #{Gitlab.config.git_base_path}" + ] + + # Gitlab repositories puts "Restoring repositories:" - project = Project.all.map { |n| [n.path, n.path_to_repo] } - project << ["gitolite-admin.git", File.join(File.dirname(project.first.second), "gitolite-admin.git")] - project.each do |project| - print "- Restoring repository #{project.first}... " - FileUtils.rm_rf(project.second) if File.dirname(project.second) # delete old stuff - if Kernel.system("cd #{File.dirname(project.second)} > /dev/null 2>&1 && git clone --bare #{backup_path_repo}/#{project.first}.bundle #{project.first}.git > /dev/null 2>&1") - permission_commands = [ - "sudo chmod -R g+rwX #{Gitlab.config.git_base_path}", - "sudo chown -R #{Gitlab.config.ssh_user}:#{Gitlab.config.ssh_user} #{Gitlab.config.git_base_path}" - ] - permission_commands.each { |command| Kernel.system(command) } + Project.all.each do |project| + + print "- Restoring repository '#{project.path_with_namespace}.git'... " + FileUtils.rm_rf(project.path_to_repo) if File.dirname(project.path_to_repo) + + if repos_in_backup.include?(project.path_with_namespace + ".bundle") # true: we can restore the repo from backup + + namespace = project.namespace? ? project.namespace.path : "/" + path_to_bundle = File.join(backup_path_repo, project.path_with_namespace + ".bundle") + path_to_repo_base = File.join(Gitlab.config.git_base_path, namespace) + + FileUtils.mkdir_p(project.path_to_repo) until Dir.exists?(path_to_repo_base) + + if Kernel.system("git clone --bare #{path_to_bundle} #{project.path_to_repo} > /dev/null 2>&1") + puts "[DONE]".green + else + puts "[FAILED]".red + end + + else # false: need to create a new empty repo + if Grit::Repo.init_bare(project.path_to_repo) + puts "[DONE]".green + else + puts "[FAILED]".red + end + end + end + + # Non gitlab managed repositories + [ "gitolite-admin.git" ].each do |repository| + path_to_bundle = File.join(backup_path_repo, repository + ".bundle") + path_to_restore_point = File.join(Gitlab.config.git_base_path, repository) + + print "- Restoring repository '#{repository}'... " + FileUtils.rm_rf(path_to_restore_point) if File.dirname(path_to_restore_point) + + if Kernel.system("git clone --bare #{path_to_bundle} #{path_to_restore_point} > /dev/null 2>&1") puts "[DONE]".green else puts "[FAILED]".red end end + # Fixing repository permissions + permission_commands.each { |command| Kernel.system(command) } end ###################################### DB ###################################### -- GitLab From 632e6f78be53bd0b33d3c4fd63e94615ada172b1 Mon Sep 17 00:00:00 2001 From: moregeek Date: Sat, 22 Dec 2012 07:26:43 +0100 Subject: [PATCH 4/4] add: gitolite testing repository and gitolite compile command --- lib/tasks/gitlab/backup.rake | 37 +++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 7f85f07283f..3e837442183 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -55,6 +55,12 @@ namespace :gitlab do # Restore backup of GitLab system desc "GITLAB | Restore a previously created backup" task :restore => :environment do + + if Process.uid != 0 + puts "Please run the restore as root user".red + exit 1 + end + Dir.chdir(Gitlab.config.backup.path) # check for existing backups in the backup dir @@ -137,15 +143,19 @@ namespace :gitlab do end # Non gitlab managed repositories - [ "gitolite-admin.git" ].each do |repository| + [ "gitolite-admin.git", "testing.git" ].each do |repository| path_to_repo = File.join(Gitlab.config.git_base_path, repository) path_to_bundle = File.join(backup_path_repo, repository + ".bundle") print "- Dumping repository '#{repository}'... " - if Kernel.system("cd #{path_to_repo} > /dev/null 2>&1 && git bundle create #{path_to_bundle} --all > /dev/null 2>&1") - puts "[DONE]".green + if Grit::Repo.new(path_to_repo).head_count == 0 + puts "[SKIPPING]".yellow else - puts "[FAILED]".red + if Kernel.system("cd #{path_to_repo} > /dev/null 2>&1 && git bundle create #{path_to_bundle} --all > /dev/null 2>&1") + puts "[DONE]".green + else + puts "[FAILED]".red + end end end end # create @@ -155,7 +165,8 @@ namespace :gitlab do repos_in_backup = Dir.chdir(backup_path_repo) && Dir.glob("{*.bundle,*/*.bundle}") permission_commands = [ "sudo chmod -R g+rwX #{Gitlab.config.git_base_path}", - "sudo chown -R #{Gitlab.config.ssh_user}:#{Gitlab.config.ssh_user} #{Gitlab.config.git_base_path}" + "sudo chown -R #{Gitlab.config.ssh_user}:#{Gitlab.config.ssh_user} #{Gitlab.config.git_base_path}", + "sudo -u git -Hi /home/git/bin/gitolite compile" # recreates gitolite access files in repositories ] # Gitlab repositories @@ -189,17 +200,25 @@ namespace :gitlab do end # |project| # Non gitlab managed repositories - [ "gitolite-admin.git" ].each do |repository| + [ "gitolite-admin.git", "testing.git" ].each do |repository| path_to_bundle = File.join(backup_path_repo, repository + ".bundle") path_to_restore_point = File.join(Gitlab.config.git_base_path, repository) print "- Restoring repository '#{repository}'... " FileUtils.rm_rf(path_to_restore_point) if File.dirname(path_to_restore_point) - if Kernel.system("git clone --bare #{path_to_bundle} #{path_to_restore_point} > /dev/null 2>&1") - puts "[DONE]".green + if File.exists?(path_to_bundle) + if Kernel.system("git clone --bare #{path_to_bundle} #{path_to_restore_point} > /dev/null 2>&1") + puts "[DONE]".green + else + puts "[FAILED]".red + end else - puts "[FAILED]".red + if Grit::Repo.init_bare(path_to_restore_point) + puts "[DONE]".green + else + puts "[FAILED]".red + end end end -- GitLab