From 45b6103997ef07644d78d603938d357a2b3d035a Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Mon, 24 Dec 2012 20:01:49 +0200 Subject: [PATCH 1/7] Update projects in gitolite after namespace moved. Added rake task to cleanup garbage from gitolite --- app/models/namespace.rb | 11 +++++++++ app/roles/namespaced_project.rb | 2 +- lib/tasks/gitlab/gitolite_rebuild.rake | 33 ++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 8c90f5aee26..96f8f291451 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -27,10 +27,13 @@ class Namespace < ActiveRecord::Base after_create :ensure_dir_exist after_update :move_dir + after_commit :update_gitolite, on: :update, if: :require_update_gitolite after_destroy :rm_dir scope :root, where('type IS NULL') + attr_accessor :require_update_gitolite + def self.search query where("name LIKE :query OR path LIKE :query", query: "%#{query}%") end @@ -62,10 +65,18 @@ class Namespace < ActiveRecord::Base if system("mv #{old_path} #{new_path}") send_update_instructions + @require_update_gitolite = true + else + raise "Namespace move error #{old_path} #{new_path}" end end end + def update_gitolite + @require_update_gitolite = false + projects.each(&:update_repository) + end + def rm_dir dir_path = File.join(Gitlab.config.gitolite.repos_path, path) system("rm -rf #{dir_path}") diff --git a/app/roles/namespaced_project.rb b/app/roles/namespaced_project.rb index 8656890a456..dbd533f8494 100644 --- a/app/roles/namespaced_project.rb +++ b/app/roles/namespaced_project.rb @@ -24,7 +24,7 @@ module NamespacedProject save! end rescue Gitlab::ProjectMover::ProjectMoveError => ex - raise TransferError.new(ex.message) + raise Project::TransferError.new(ex.message) end def name_with_namespace diff --git a/lib/tasks/gitlab/gitolite_rebuild.rake b/lib/tasks/gitlab/gitolite_rebuild.rake index fce10eb5b69..41b48f73d8f 100644 --- a/lib/tasks/gitlab/gitolite_rebuild.rake +++ b/lib/tasks/gitlab/gitolite_rebuild.rake @@ -20,5 +20,38 @@ namespace :gitlab do end puts "Done with keys" end + + desc "GITLAB | Cleanup gitolite config" + task :cleanup => :environment do + warn_user_is_not_gitlab + + real_repos = Project.all.map(&:path_with_namespace) + real_repos << "gitolite-admin" + real_repos << "@all" + + remove_flag = ENV['REMOVE'] + + puts "Looking for repositories to remove... " + Gitlab::GitoliteConfig.new.apply do |config| + all_repos = [] + garbage_repos = [] + + all_repos = config.conf.repos.keys + garbage_repos = all_repos - real_repos + + garbage_repos.each do |repo_name| + if remove_flag + config.conf.rm_repo(repo_name) + print "to remove...".red + end + + puts repo_name.red + end + end + + unless remove_flag + puts "To cleanup repositories run this command with REMOVE=true".yellow + end + end end end -- GitLab From 19cb29e448f8959d86dde38f8f85ad01c3e1bb72 Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Mon, 24 Dec 2012 21:27:12 +0200 Subject: [PATCH 2/7] Fix MR count for group scope --- app/views/layouts/group.html.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/layouts/group.html.haml b/app/views/layouts/group.html.haml index d40d9525bb8..f47e8b3e9ff 100644 --- a/app/views/layouts/group.html.haml +++ b/app/views/layouts/group.html.haml @@ -15,7 +15,7 @@ = nav_link(path: 'groups#merge_requests') do = link_to merge_requests_group_path(@group) do Merge Requests - %span.count= current_user.cared_merge_requests.of_group(@group).count + %span.count= current_user.cared_merge_requests.opened.of_group(@group).count = nav_link(path: 'groups#search') do = link_to "Search", search_group_path(@group) = nav_link(path: 'groups#people') do -- GitLab From c68540e9c94eed7dbe7683e47450b026c9ee3bb6 Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Wed, 26 Dec 2012 19:56:56 +0200 Subject: [PATCH 3/7] Fix resque for postgres --- config/initializers/4_resque.rb | 2 ++ lib/tasks/resque.rake | 1 + 2 files changed, 3 insertions(+) diff --git a/config/initializers/4_resque.rb b/config/initializers/4_resque.rb index 419dbe06061..03c2b785fae 100644 --- a/config/initializers/4_resque.rb +++ b/config/initializers/4_resque.rb @@ -27,3 +27,5 @@ Resque::Server.use Authentication # Mailer Resque::Mailer.excluded_environments = [] + +Resque.before_fork = Proc.new { ActiveRecord::Base.establish_connection } diff --git a/lib/tasks/resque.rake b/lib/tasks/resque.rake index 0825324a424..bf031403db8 100644 --- a/lib/tasks/resque.rake +++ b/lib/tasks/resque.rake @@ -4,6 +4,7 @@ task "resque:setup" => :environment do Resque.after_fork do Resque.redis.client.reconnect end + Resque.before_fork = Proc.new { ActiveRecord::Base.establish_connection } end desc "Alias for resque:work (To run workers on Heroku)" -- GitLab From 2edb16612f9491af5c1e0e0b6384661fb972b7cf Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Tue, 25 Dec 2012 07:14:05 +0300 Subject: [PATCH 4/7] Cleanup service tasks --- lib/tasks/gitlab/cleanup.rake | 128 +++++++++++++++++++++++++ lib/tasks/gitlab/gitolite_rebuild.rake | 33 ------- 2 files changed, 128 insertions(+), 33 deletions(-) create mode 100644 lib/tasks/gitlab/cleanup.rake diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake new file mode 100644 index 00000000000..2a0ffe0f4bd --- /dev/null +++ b/lib/tasks/gitlab/cleanup.rake @@ -0,0 +1,128 @@ +namespace :gitlab do + namespace :cleanup do + desc "GITLAB | Cleanup | Clean gitolite config" + task :config => :environment do + warn_user_is_not_gitlab + + real_repos = Project.all.map(&:path_with_namespace) + real_repos << "gitolite-admin" + real_repos << "@all" + + remove_flag = ENV['REMOVE'] + + puts "Looking for repositories to remove... " + Gitlab::GitoliteConfig.new.apply do |config| + all_repos = [] + garbage_repos = [] + + all_repos = config.conf.repos.keys + garbage_repos = all_repos - real_repos + + garbage_repos.each do |repo_name| + if remove_flag + config.conf.rm_repo(repo_name) + print "to remove...".red + end + + puts repo_name.red + end + end + + unless remove_flag + puts "To cleanup repositories run this command with REMOVE=true".yellow + end + end + + desc "GITLAB | Cleanup | Clean namespaces" + task :dirs => :environment do + warn_user_is_not_gitlab + remove_flag = ENV['REMOVE'] + + + namespaces = Namespace.pluck(:path) + git_base_path = Gitlab.config.gitolite.repos_path + all_dirs = Dir.glob(git_base_path + '/*') + + puts git_base_path.yellow + puts "Looking for directories to remove... " + + all_dirs.reject! do |dir| + # skip if git repo + dir =~ /.git$/ + end + + all_dirs.reject! do |dir| + dir_name = File.basename dir + + # skip if namespace present + namespaces.include?(dir_name) + end + + all_dirs.each do |dir_path| + + if remove_flag + if FileUtils.rm_rf dir_path + puts "Removed...#{dir_path}".red + else + puts "Cannot remove #{dir_path}".red + end + else + puts "Can be removed: #{dir_path}".red + end + end + + unless remove_flag + puts "To cleanup this directories run this command with REMOVE=true".yellow + end + end + + desc "GITLAB | Cleanup | Clean respositories" + task :repos => :environment do + warn_user_is_not_gitlab + remove_flag = ENV['REMOVE'] + + git_base_path = Gitlab.config.gitolite.repos_path + all_dirs = Dir.glob(git_base_path + '/*') + + global_projects = Project.where(namespace_id: nil).pluck(:path) + + puts git_base_path.yellow + puts "Looking for global repos to remove... " + + # skip non git repo + all_dirs.select! do |dir| + dir =~ /.git$/ + end + + # skip existing repos + all_dirs.reject! do |dir| + repo_name = File.basename dir + path = repo_name.gsub(/\.git$/, "") + global_projects.include?(path) + end + + # skip gitolite admin + all_dirs.reject! do |dir| + repo_name = File.basename dir + repo_name == 'gitolite-admin.git' + end + + + all_dirs.each do |dir_path| + if remove_flag + if FileUtils.rm_rf dir_path + puts "Removed...#{dir_path}".red + else + puts "Cannot remove #{dir_path}".red + end + else + puts "Can be removed: #{dir_path}".red + end + end + + unless remove_flag + puts "To cleanup this directories run this command with REMOVE=true".yellow + end + end + end +end diff --git a/lib/tasks/gitlab/gitolite_rebuild.rake b/lib/tasks/gitlab/gitolite_rebuild.rake index 41b48f73d8f..fce10eb5b69 100644 --- a/lib/tasks/gitlab/gitolite_rebuild.rake +++ b/lib/tasks/gitlab/gitolite_rebuild.rake @@ -20,38 +20,5 @@ namespace :gitlab do end puts "Done with keys" end - - desc "GITLAB | Cleanup gitolite config" - task :cleanup => :environment do - warn_user_is_not_gitlab - - real_repos = Project.all.map(&:path_with_namespace) - real_repos << "gitolite-admin" - real_repos << "@all" - - remove_flag = ENV['REMOVE'] - - puts "Looking for repositories to remove... " - Gitlab::GitoliteConfig.new.apply do |config| - all_repos = [] - garbage_repos = [] - - all_repos = config.conf.repos.keys - garbage_repos = all_repos - real_repos - - garbage_repos.each do |repo_name| - if remove_flag - config.conf.rm_repo(repo_name) - print "to remove...".red - end - - puts repo_name.red - end - end - - unless remove_flag - puts "To cleanup repositories run this command with REMOVE=true".yellow - end - end end end -- GitLab From 91405d1f989010d7e5676b2707e2797b16b62ecd Mon Sep 17 00:00:00 2001 From: "Phil. Austermann" Date: Thu, 27 Dec 2012 11:03:54 +0100 Subject: [PATCH 5/7] - Fix PostgreSql Migration issue to 4.0 (fixes https://github.com/gitlabhq/gitlabhq/issues/2398) --- db/migrate/20121218164840_move_noteable_commit_to_own_field.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/migrate/20121218164840_move_noteable_commit_to_own_field.rb b/db/migrate/20121218164840_move_noteable_commit_to_own_field.rb index 6f2da4136a3..f6b97390b93 100644 --- a/db/migrate/20121218164840_move_noteable_commit_to_own_field.rb +++ b/db/migrate/20121218164840_move_noteable_commit_to_own_field.rb @@ -5,7 +5,7 @@ class MoveNoteableCommitToOwnField < ActiveRecord::Migration Note.where(noteable_type: 'Commit').update_all('commit_id = noteable_id') if ActiveRecord::Base.connection.adapter_name == 'PostgreSQL' - Note.where("noteable_type != 'Commit'").update_all('new_noteable_id = CAST (noteable_id AS INTEGER)') + Note.where("noteable_type != 'Commit'").update_all('new_noteable_id = CAST (CASE noteable_id WHEN \'\' THEN NULL ELSE noteable_id END AS INTEGER)') else Note.where("noteable_type != 'Commit'").update_all('new_noteable_id = noteable_id') end -- GitLab From 708a0d421e42522c6f10ff272d40db36b2205d96 Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Fri, 28 Dec 2012 06:14:05 +0300 Subject: [PATCH 6/7] Fixed and improved enable_naamespace migration task --- app/models/namespace.rb | 13 +++- app/observers/user_observer.rb | 2 +- lib/tasks/gitlab/enable_namespaces.rake | 84 ++++++++++++++++++++----- 3 files changed, 79 insertions(+), 20 deletions(-) diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 96f8f291451..d34e5a99c79 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -51,8 +51,17 @@ class Namespace < ActiveRecord::Base end def ensure_dir_exist - namespace_dir_path = File.join(Gitlab.config.gitolite.repos_path, path) - system("mkdir -m 770 #{namespace_dir_path}") unless File.exists?(namespace_dir_path) + unless dir_exists? + system("mkdir -m 770 #{namespace_full_path}") + end + end + + def dir_exists? + File.exists?(namespace_full_path) + end + + def namespace_full_path + @namespace_full_path ||= File.join(Gitlab.config.gitolite.repos_path, path) end def move_dir diff --git a/app/observers/user_observer.rb b/app/observers/user_observer.rb index 09b3c1d622f..73a1d00ca3b 100644 --- a/app/observers/user_observer.rb +++ b/app/observers/user_observer.rb @@ -14,7 +14,7 @@ class UserObserver < ActiveRecord::Observer if user.namespace user.namespace.update_attributes(path: user.username) else - user.create_namespace!(path: user.username, name: user.name) + user.create_namespace!(path: user.username, name: user.username) end end end diff --git a/lib/tasks/gitlab/enable_namespaces.rake b/lib/tasks/gitlab/enable_namespaces.rake index 1be9ba6469d..ee80c0905cb 100644 --- a/lib/tasks/gitlab/enable_namespaces.rake +++ b/lib/tasks/gitlab/enable_namespaces.rake @@ -1,34 +1,87 @@ namespace :gitlab do desc "GITLAB | Enable usernames and namespaces for user projects" task enable_namespaces: :environment do - print "\nUsernames for users:".yellow + warn_user_is_not_gitlab + migrate_user_namespaces + migrate_groups + migrate_projects + + puts "Rebuild Gitolite ... " + gitolite = Gitlab::Gitolite.new + gitolite.update_repositories(Project.where('namespace_id IS NOT NULL')) + puts "... #{"done".green}" + end + + def migrate_user_namespaces + puts "\nGenerate usernames for users without one: ".blue User.find_each(batch_size: 500) do |user| - next if user.namespace + if user.namespace + print '-'.cyan + next + end - User.transaction do - username = user.email.match(/^[^@]*/)[0] - if user.update_attributes!(username: username) + username = if user.username.present? + # if user already has username filled + user.username + else + build_username(user) + end + + begin + User.transaction do + user.update_attributes!(username: username) print '.'.green - else - print 'F'.red end + rescue + print 'F'.red end end + puts "\nDone" + end + + def build_username(user) + username = nil + + # generate username + username = user.email.match(/^[^@]*/)[0] + username.gsub!("+", ".") + + # return username if no mathes + return username unless User.find_by_username(username) + + # look for same username + (1..10).each do |i| + suffixed_username = "#{username}#{i}" + + return suffixed_username unless User.find_by_username(suffixed_username) + end + end - print "\n\nDirs for groups:".yellow + def migrate_groups + puts "\nCreate directories for groups: ".blue Group.find_each(batch_size: 500) do |group| - if group.ensure_dir_exist - print '.'.green - else + begin + if group.dir_exists? + print '-'.cyan + else + if group.ensure_dir_exist + print '.'.green + else + print 'F'.red + end + end + rescue print 'F'.red end end + puts "\nDone" + end - print "\n\nMove projects from groups under groups dirs:".yellow + def migrate_projects git_path = Gitlab.config.gitolite.repos_path - + puts "\nMove projects in groups into respective directories ... ".blue Project.where('namespace_id IS NOT NULL').find_each(batch_size: 500) do |project| next unless project.group @@ -59,9 +112,6 @@ namespace :gitlab do end end - print "\n\nRebuild gitolite:".yellow - gitolite = Gitlab::Gitolite.new - gitolite.update_repositories(Project.where('namespace_id IS NOT NULL')) - puts "\n" + puts "\nDone" end end -- GitLab From 91edf7f959b347335a174d365f19b711ccc2e50b Mon Sep 17 00:00:00 2001 From: Paul Canham Date: Fri, 28 Dec 2012 15:33:49 +0000 Subject: [PATCH 7/7] Update doc/install/installation.md --- doc/install/installation.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/install/installation.md b/doc/install/installation.md index 718e4cf6419..f42d5470b12 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -245,12 +245,12 @@ Make sure to update username/password in config/database.yml. sudo gem install charlock_holmes --version '0.6.9' - # For mysql db - sudo -u gitlab -H bundle install --deployment --without development test postgres - - # Or For postgres db + # For Mysql db sudo -u gitlab -H bundle install --deployment --without development test mysql + # Or For PostgreSQL db + sudo -u gitlab -H bundle install --deployment --without development test postgres + ## Configure Git GitLab needs to be able to commit and push changes to Gitolite. In order to do -- GitLab