diff --git a/Gemfile b/Gemfile index 2e465f8ced7dd5c433865e9e41fdc297caeee682..f36e2e38d6be763d020d69d217462e1f551fd51b 100644 --- a/Gemfile +++ b/Gemfile @@ -18,7 +18,7 @@ gem 'gitlab-default_value_for', '~> 3.1.1', require: 'default_value_for' gem 'mysql2', '~> 0.4.10', group: :mysql gem 'pg', '~> 1.1', group: :postgres -gem 'rugged', '~> 0.27' +gem 'rugged', '~> 0.28' gem 'grape-path-helpers', '~> 1.0' gem 'faraday', '~> 0.12' diff --git a/Gemfile.lock b/Gemfile.lock index 4d37075cdfac84d3ac12692b0fcd4e54af11d08a..1be6f2289544c90dfce858f5b389c66f92cc98ae 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -785,7 +785,7 @@ GEM rubyntlm (0.6.2) rubypants (0.2.0) rubyzip (1.2.2) - rugged (0.27.5) + rugged (0.28.0) safe_yaml (1.0.4) sanitize (4.6.6) crass (~> 1.0.2) @@ -1138,7 +1138,7 @@ DEPENDENCIES ruby-progressbar ruby_parser (~> 3.8) rubyzip (~> 1.2.2) - rugged (~> 0.27) + rugged (~> 0.28) sanitize (~> 4.6) sass (~> 3.5) sass-rails (~> 5.0.6) diff --git a/app/assets/javascripts/filtered_search/visual_token_value.js b/app/assets/javascripts/filtered_search/visual_token_value.js index 99008a4c989b40e48fdc79dfd6275197b340d149..a9d5ba8faa8d7221168abbe9c571657e1911b90b 100644 --- a/app/assets/javascripts/filtered_search/visual_token_value.js +++ b/app/assets/javascripts/filtered_search/visual_token_value.js @@ -13,9 +13,9 @@ export default class VisualTokenValue { } render(tokenValueContainer, tokenValueElement) { - const { tokenType } = this; + const { tokenType, tokenValue } = this; - if (['none', 'any'].includes(tokenType)) { + if (['none', 'any'].includes(tokenValue.toLowerCase())) { return; } diff --git a/app/assets/javascripts/lib/utils/simple_poll.js b/app/assets/javascripts/lib/utils/simple_poll.js index 473f179ad86e99048d20846334abb5db899693c6..576a9ec880cb6e52333a3a74025f58ca609442a5 100644 --- a/app/assets/javascripts/lib/utils/simple_poll.js +++ b/app/assets/javascripts/lib/utils/simple_poll.js @@ -1,10 +1,10 @@ -export default (fn, interval = 2000, timeout = 60000) => { +export default (fn, { interval = 2000, timeout = 60000 } = {}) => { const startTime = Date.now(); return new Promise((resolve, reject) => { const stop = arg => (arg instanceof Error ? reject(arg) : resolve(arg)); const next = () => { - if (Date.now() - startTime < timeout) { + if (timeout === 0 || Date.now() - startTime < timeout) { setTimeout(fn.bind(null, next, stop), interval); } else { reject(new Error('SIMPLE_POLL_TIMEOUT')); diff --git a/app/assets/javascripts/mirrors/ssh_mirror.js b/app/assets/javascripts/mirrors/ssh_mirror.js index 5bdf5d6277a0b55782354134ffc7138624c6d6ac..547c078ec55f35dcfe8c4bda1ecd9cdddf78db2f 100644 --- a/app/assets/javascripts/mirrors/ssh_mirror.js +++ b/app/assets/javascripts/mirrors/ssh_mirror.js @@ -20,6 +20,7 @@ export default class SSHMirror { this.$btnDetectHostKeys = this.$form.find('.js-detect-host-keys'); this.$btnSSHHostsShowAdvanced = this.$form.find('.btn-show-advanced'); this.$dropdownAuthType = this.$form.find('.js-mirror-auth-type'); + this.$hiddenAuthType = this.$form.find('.js-hidden-mirror-auth-type'); this.$wellAuthTypeChanging = this.$form.find('.js-well-changing-auth'); this.$wellPasswordAuth = this.$form.find('.js-well-password-auth'); @@ -167,6 +168,7 @@ export default class SSHMirror { this.$wellPasswordAuth.collapse('hide'); this.$wellSSHAuth.collapse('hide'); + this.updateHiddenAuthType(selectedAuthType); // This request should happen only if selected Auth type was SSH // and SSH Public key was not present on page load @@ -234,6 +236,12 @@ export default class SSHMirror { toggleAuthWell(authType) { this.$wellPasswordAuth.collapse(authType === AUTH_METHOD.PASSWORD ? 'show' : 'hide'); this.$wellSSHAuth.collapse(authType === AUTH_METHOD.SSH ? 'show' : 'hide'); + this.updateHiddenAuthType(authType); + } + + updateHiddenAuthType(authType) { + this.$hiddenAuthType.val(authType); + this.$hiddenAuthType.prop('disabled', authType === AUTH_METHOD.SSH); } /** diff --git a/app/assets/javascripts/vue_merge_request_widget/components/states/ready_to_merge.vue b/app/assets/javascripts/vue_merge_request_widget/components/states/ready_to_merge.vue index 8e043ed50c95a62f0a070f31e60802dcb4b5b279..bb76eb1030d3b7e7904667fe01a3627ee316d677 100644 --- a/app/assets/javascripts/vue_merge_request_widget/components/states/ready_to_merge.vue +++ b/app/assets/javascripts/vue_merge_request_widget/components/states/ready_to_merge.vue @@ -165,9 +165,12 @@ export default { }); }, initiateMergePolling() { - simplePoll((continuePolling, stopPolling) => { - this.handleMergePolling(continuePolling, stopPolling); - }); + simplePoll( + (continuePolling, stopPolling) => { + this.handleMergePolling(continuePolling, stopPolling); + }, + { timeout: 0 }, + ); }, handleMergePolling(continuePolling, stopPolling) { this.service @@ -198,6 +201,7 @@ export default { }) .catch(() => { new Flash(__('Something went wrong while merging this merge request. Please try again.')); // eslint-disable-line + stopPolling(); }); }, initiateRemoveSourceBranchPolling() { diff --git a/app/assets/stylesheets/pages/settings.scss b/app/assets/stylesheets/pages/settings.scss index 4f9d96da4bdd76203961cd168f2e3759ea8b807b..54126577f93e971de2ee34fd6319c16d373e87d4 100644 --- a/app/assets/stylesheets/pages/settings.scss +++ b/app/assets/stylesheets/pages/settings.scss @@ -23,7 +23,10 @@ } .settings { - border-bottom: 1px solid $gray-darker; + // border-top for each item except the top one + + .settings { + border-top: 1px solid $border-color; + } &:first-of-type { margin-top: 10px; diff --git a/app/controllers/admin/projects_controller.rb b/app/controllers/admin/projects_controller.rb index 550f29a58d2cdcbf792e28ad807f88b7bec7c56e..3fa61c7b117d4da71cf9b179523efac6a6d79d4b 100644 --- a/app/controllers/admin/projects_controller.rb +++ b/app/controllers/admin/projects_controller.rb @@ -15,7 +15,7 @@ class Admin::ProjectsController < Admin::ApplicationController format.html format.json do render json: { - html: view_to_html_string("admin/projects/_projects", locals: { projects: @projects }) + html: view_to_html_string("admin/projects/_projects", projects: @projects) } end end diff --git a/app/controllers/dashboard/projects_controller.rb b/app/controllers/dashboard/projects_controller.rb index b044affd4e86b8cfa183291291b0246c84931538..0a47736cad8ddd9e2a0f7b6e928d8794b6c1a3e7 100644 --- a/app/controllers/dashboard/projects_controller.rb +++ b/app/controllers/dashboard/projects_controller.rb @@ -26,7 +26,7 @@ class Dashboard::ProjectsController < Dashboard::ApplicationController end format.json do render json: { - html: view_to_html_string("dashboard/projects/_projects", locals: { projects: @projects }) + html: view_to_html_string("dashboard/projects/_projects", projects: @projects) } end end @@ -43,7 +43,7 @@ class Dashboard::ProjectsController < Dashboard::ApplicationController format.html format.json do render json: { - html: view_to_html_string("dashboard/projects/_projects", locals: { projects: @projects }) + html: view_to_html_string("dashboard/projects/_projects", projects: @projects) } end end diff --git a/app/controllers/explore/projects_controller.rb b/app/controllers/explore/projects_controller.rb index f3d76c5a4788b06b7388b4d2d1fceb365c5e303e..ef86d5f981ac8fe32db5c7b60e311d732ea29cba 100644 --- a/app/controllers/explore/projects_controller.rb +++ b/app/controllers/explore/projects_controller.rb @@ -15,7 +15,7 @@ class Explore::ProjectsController < Explore::ApplicationController format.html format.json do render json: { - html: view_to_html_string("explore/projects/_projects", locals: { projects: @projects }) + html: view_to_html_string("explore/projects/_projects", projects: @projects) } end end @@ -30,7 +30,7 @@ class Explore::ProjectsController < Explore::ApplicationController format.html format.json do render json: { - html: view_to_html_string("explore/projects/_projects", locals: { projects: @projects }) + html: view_to_html_string("explore/projects/_projects", projects: @projects) } end end @@ -44,7 +44,7 @@ class Explore::ProjectsController < Explore::ApplicationController format.html format.json do render json: { - html: view_to_html_string("explore/projects/_projects", locals: { projects: @projects }) + html: view_to_html_string("explore/projects/_projects", projects: @projects) } end end diff --git a/app/controllers/projects/settings/operations_controller.rb b/app/controllers/projects/settings/operations_controller.rb index 7276964b6e1653d63cb758904093bc205dc7fcb8..1fafc33e9171e976a6c12d9056a156abe4540956 100644 --- a/app/controllers/projects/settings/operations_controller.rb +++ b/app/controllers/projects/settings/operations_controller.rb @@ -3,7 +3,6 @@ module Projects module Settings class OperationsController < Projects::ApplicationController - before_action :check_license before_action :authorize_update_environment! helper_method :error_tracking_setting @@ -65,10 +64,6 @@ module Projects ] } end - - def check_license - render_404 unless helpers.settings_operations_available? - end end end end diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index c5035797621ba5eba0bda5607c34047d9dcac0a6..da5a82af2fde015dc4e6f61bee5e907e10124d36 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -7,7 +7,7 @@ class ApplicationSetting < ActiveRecord::Base include IgnorableColumn include ChronicDurationAttribute - add_authentication_token_field :runners_registration_token, encrypted: -> { Feature.enabled?(:application_settings_tokens_optional_encryption) ? :optional : :required } + add_authentication_token_field :runners_registration_token, encrypted: -> { Feature.enabled?(:application_settings_tokens_optional_encryption, default_enabled: true) ? :optional : :required } add_authentication_token_field :health_check_access_token DOMAIN_LIST_SEPARATOR = %r{\s*[,;]\s* # comma or semicolon, optionally surrounded by whitespace diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index ce26ee168ef6562517846f70ccf8b216e386b0e4..43f040a91aeea7b08a31705ce97979d286c279b6 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -10,7 +10,7 @@ module Ci include FromUnion include TokenAuthenticatable - add_authentication_token_field :token, encrypted: -> { Feature.enabled?(:ci_runners_tokens_optional_encryption) ? :optional : :required } + add_authentication_token_field :token, encrypted: -> { Feature.enabled?(:ci_runners_tokens_optional_encryption, default_enabled: true) ? :optional : :required } enum access_level: { not_protected: 0, diff --git a/app/models/group.rb b/app/models/group.rb index 495bfe04499116de945fc0737efe882396fdc305..c77586c4cdc8800721b5b8d3cbe1d7261a4c7a27 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -56,7 +56,7 @@ class Group < Namespace validates :two_factor_grace_period, presence: true, numericality: { greater_than_or_equal_to: 0 } - add_authentication_token_field :runners_token, encrypted: -> { Feature.enabled?(:groups_tokens_optional_encryption) ? :optional : :required } + add_authentication_token_field :runners_token, encrypted: -> { Feature.enabled?(:groups_tokens_optional_encryption, default_enabled: true) ? :optional : :required } after_create :post_create_hook after_destroy :post_destroy_hook diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb index 351a662ae83cbc6c3962528b942056f5efa479c5..6c1592604d3dd00126ebc3d62f08c74cf6864cc6 100644 --- a/app/models/merge_request_diff.rb +++ b/app/models/merge_request_diff.rb @@ -298,6 +298,11 @@ class MergeRequestDiff < ActiveRecord::Base private + def encode_in_base64?(diff_text) + (diff_text.encoding == Encoding::BINARY && !diff_text.ascii_only?) || + diff_text.include?("\0") + end + def create_merge_request_diff_files(diffs) rows = if has_attribute?(:external_diff) && Gitlab.config.external_diffs.enabled @@ -350,7 +355,7 @@ class MergeRequestDiff < ActiveRecord::Base diff_hash.tap do |hash| diff_text = hash[:diff] - if diff_text.encoding == Encoding::BINARY && !diff_text.ascii_only? + if encode_in_base64?(diff_text) hash[:binary] = true hash[:diff] = [diff_text].pack('m0') end diff --git a/app/models/note.rb b/app/models/note.rb index 1578ae9c4cce4b89941b72db7c8172f016a9dabd..2c9980b1a0d4dff54a2eab3229deea0e692800cc 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -313,6 +313,14 @@ class Note < ActiveRecord::Base !system? end + # Since we're using `updated_at` as `last_edited_at`, it could be touched by transforming / resolving a note. + # This makes sure it is only marked as edited when the note body is updated. + def edited? + return false if updated_by.blank? + + super + end + def cross_reference_not_visible_for?(user) cross_reference? && !all_referenced_mentionables_allowed?(user) end diff --git a/app/models/project.rb b/app/models/project.rb index 4cc13f372c166394b926190abc5e0a243c01d3c4..7d6f7fd2c581325dc4b0dfa268d37a4e5568448e 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -85,7 +85,7 @@ class Project < ActiveRecord::Base default_value_for :snippets_enabled, gitlab_config_features.snippets default_value_for :only_allow_merge_if_all_discussions_are_resolved, false - add_authentication_token_field :runners_token, encrypted: -> { Feature.enabled?(:projects_tokens_optional_encryption) ? :optional : :required } + add_authentication_token_field :runners_token, encrypted: -> { Feature.enabled?(:projects_tokens_optional_encryption, default_enabled: true) ? :optional : :required } before_validation :mark_remote_mirrors_for_removal, if: -> { RemoteMirror.table_exists? } diff --git a/app/views/projects/mirrors/_authentication_method.html.haml b/app/views/projects/mirrors/_authentication_method.html.haml index 293a2e3ebfe871606e67129e8a4e44ce4be180e2..ef6db07a1bb3b57a36216e26ed36d71f82322da2 100644 --- a/app/views/projects/mirrors/_authentication_method.html.haml +++ b/app/views/projects/mirrors/_authentication_method.html.haml @@ -9,6 +9,7 @@ = f.select :auth_method, options_for_select(auth_options, mirror.auth_method), {}, { class: "form-control js-mirror-auth-type qa-authentication-method" } + = f.hidden_field :auth_method, value: "password", class: "js-hidden-mirror-auth-type" .form-group .collapse.js-well-changing-auth diff --git a/app/views/projects/settings/operations/_error_tracking.html.haml b/app/views/projects/settings/operations/_error_tracking.html.haml index 6b15331db01e1b74a3ecdb0772ef995a521f3c3b..451a79becc3ea1a94ac7e51c5821edc43a08b0eb 100644 --- a/app/views/projects/settings/operations/_error_tracking.html.haml +++ b/app/views/projects/settings/operations/_error_tracking.html.haml @@ -2,7 +2,7 @@ - setting = error_tracking_setting -%section.settings.expanded.border-0.no-animate +%section.settings.expanded.no-animate .settings-header %h4 = _('Error Tracking') diff --git a/app/views/projects/settings/operations/show.html.haml b/app/views/projects/settings/operations/show.html.haml index 2822debe426b80f3a68220d0a028fa404dd03183..6f777305a549a4b71081766fe07e368bea517af3 100644 --- a/app/views/projects/settings/operations/show.html.haml +++ b/app/views/projects/settings/operations/show.html.haml @@ -2,5 +2,6 @@ - page_title _('Operations Settings') - breadcrumb_title _('Operations Settings') += render_if_exists 'projects/settings/operations/incidents' = render 'projects/settings/operations/error_tracking', expanded: true = render_if_exists 'projects/settings/operations/tracing' diff --git a/changelogs/unreleased/56970-fix-mr-stuck-loading-on-error.yml b/changelogs/unreleased/56970-fix-mr-stuck-loading-on-error.yml new file mode 100644 index 0000000000000000000000000000000000000000..f86c77d0e2455fc944e33def27fa440598096b55 --- /dev/null +++ b/changelogs/unreleased/56970-fix-mr-stuck-loading-on-error.yml @@ -0,0 +1,5 @@ +--- +title: Disable timeout on merge request merging poll +merge_request: 25988 +author: +type: fixed diff --git a/changelogs/unreleased/57330-fix-comment-edited.yml b/changelogs/unreleased/57330-fix-comment-edited.yml new file mode 100644 index 0000000000000000000000000000000000000000..68cf6c03d4c9293d7025dd2365135b6be8304a2c --- /dev/null +++ b/changelogs/unreleased/57330-fix-comment-edited.yml @@ -0,0 +1,5 @@ +--- +title: Fix notes being marked as edited after resolving +merge_request: 26143 +author: +type: fixed diff --git a/changelogs/unreleased/fix-projects-partial-locals.yml b/changelogs/unreleased/fix-projects-partial-locals.yml new file mode 100644 index 0000000000000000000000000000000000000000..7e2cc008105bd80c0365077bddb875dc2a95b72d --- /dev/null +++ b/changelogs/unreleased/fix-projects-partial-locals.yml @@ -0,0 +1,5 @@ +--- +title: Fix undefined variable error on json project views +merge_request: 26297 +author: +type: fixed diff --git a/changelogs/unreleased/sh-handle-null-bytes-in-merge-request-diffs.yml b/changelogs/unreleased/sh-handle-null-bytes-in-merge-request-diffs.yml new file mode 100644 index 0000000000000000000000000000000000000000..01b6b08b61bc95ad7b74fcb5d0fc94fcc9a982c2 --- /dev/null +++ b/changelogs/unreleased/sh-handle-null-bytes-in-merge-request-diffs.yml @@ -0,0 +1,5 @@ +--- +title: Fix error creating a merge request when diff includes a null byte +merge_request: 26190 +author: +type: fixed diff --git a/doc/administration/raketasks/storage.md b/doc/administration/raketasks/storage.md index 7ad38abe4f554d7a71ccb983bd0ff87933dff56a..c39fef907dba376109570cd11b861076f0e03154 100644 --- a/doc/administration/raketasks/storage.md +++ b/doc/administration/raketasks/storage.md @@ -34,17 +34,59 @@ export ID_FROM=20 export ID_TO=50 ``` -You can monitor the progress in the _Admin > Monitoring > Background jobs_ screen. -There is a specific Queue you can watch to see how long it will take to finish: **project_migrate_hashed_storage** +You can monitor the progress in the **Admin Area > Monitoring > Background Jobs** page. +There is a specific Queue you can watch to see how long it will take to finish: +`hashed_storage:hashed_storage_project_migrate` After it reaches zero, you can confirm every project has been migrated by running the commands bellow. If you find it necessary, you can run this migration script again to schedule missing projects. -Any error or warning will be logged in the sidekiq's log file. +Any error or warning will be logged in Sidekiq's log file. You only need the `gitlab:storage:migrate_to_hashed` rake task to migrate your repositories, but we have additional commands below that helps you inspect projects and attachments in both legacy and hashed storage. +## Rollback from Hashed storage to Legacy storage + +If you need to rollback the storage migration for any reason, you can follow the steps described here. + +NOTE: **Note:** Hashed Storage will be required in future version of GitLab. + +To prevent new projects from being created in the Hashed storage, +you need to undo the [enable hashed storage][storage-migration] changes. + +This task will schedule all your existing projects and associated attachments to be rolled back to the +Legacy storage type. + +For Omnibus installations, run the following: + +```bash +sudo gitlab-rake gitlab:storage:rollback_to_legacy +``` + +For source installations, run the following: + +```bash +sudo -u git -H bundle exec rake gitlab:storage:rollback_to_legacy RAILS_ENV=production +``` + +Both commands accept a range as environment variable: + +```bash +# to rollback any migrated project from ID 20 to 50. +export ID_FROM=20 +export ID_TO=50 +``` + +You can monitor the progress in the **Admin Area > Monitoring > Background Jobs** page. +On the **Queues** tab, you can watch the `hashed_storage:hashed_storage_project_rollback` queue to see how long the process will take to finish. + + +After it reaches zero, you can confirm every project has been rolled back by running the commands bellow. +If some projects weren't rolled back, you can run this rollback script again to schedule further rollbacks. + +Any error or warning will be logged in Sidekiq's log file. + ## List projects on Legacy storage To have a simple summary of projects using **Legacy** storage: diff --git a/doc/administration/repository_storage_types.md b/doc/administration/repository_storage_types.md index 4934aaf39f70157cd4c3d52b49af2f0c6403733b..40f7c5566ac661941b8733b2fd9ff3aa090eef36 100644 --- a/doc/administration/repository_storage_types.md +++ b/doc/administration/repository_storage_types.md @@ -2,6 +2,24 @@ > [Introduced][ce-28283] in GitLab 10.0. +Two different storage layouts can be used +to store the repositories on disk and their characteristics. + +GitLab can be configured to use one or multiple repository shard locations +that can be: + +- Mounted to the local disk +- Exposed as an NFS shared volume +- Acessed via [gitaly] on its own machine. + +In GitLab, this is configured in `/etc/gitlab/gitlab.rb` by the `git_data_dirs({})` +configuration hash. The storage layouts discussed here will apply to any shard +defined in it. + +The `default` repository shard that is available in any installations +that haven't customized it, points to the local folder: `/var/opt/gitlab/git-data`. +Anything discussed below is expected to be part of that folder. + ## Legacy Storage Legacy Storage is the storage behavior prior to version 10.0. For historical @@ -66,34 +84,7 @@ by another folder with the next 2 characters. They are both stored in a special "@hashed/#{hash[0..1]}/#{hash[2..3]}/#{hash}.wiki.git" ``` -### How to migrate to Hashed Storage - -In GitLab, go to **Admin > Settings**, find the **Repository Storage** section -and select "_Use hashed storage paths for newly created and renamed projects_". - -To migrate your existing projects to the new storage type, check the specific -[rake tasks]. - -[ce-28283]: https://gitlab.com/gitlab-org/gitlab-ce/issues/28283 -[rake tasks]: raketasks/storage.md#migrate-existing-projects-to-hashed-storage -[storage-paths]: repository_storage_types.md - -#### Rollback - -There is no automated rollback implemented. Below are the steps required to rollback -from each storage migration. - -The rollback has to be performed in the reverse order. To get into "Legacy" state, -you need to rollback Attachments first, then Project. - -Also note that if Geo is enabled, after the migration was triggered, an event is generated -to replicate the operation on any Secondary node. That means the on disk changes will also -need to be performed on these nodes as well. Database changes will propagate without issues. - -You must make sure the migration event was already processed or otherwise it may migrate -the files back to Hashed state again. - -#### Hashed object pools +### Hashed object pools For deduplication of public forks and their parent repository, objects are pooled in an object pool. These object pools are a third repository where shared objects @@ -110,36 +101,60 @@ enabled for individual projects by executing be on hashed storage, should not be a fork itself, and hashed storage should be enabled for all new projects. -##### Attachments +### How to migrate to Hashed Storage -To rollback single Attachment migration, rename `aa/bb/abcdef1234567890...` folder back to `namespace/project`. +To start a migration, enable Hashed Storage for new projects: + +1. Go to **Admin > Settings** and expand the **Repository Storage** section. +2. Select the **Use hashed storage paths for newly created and renamed projects** checkbox. -Both folder names can be generated by the `FileUploader.absolute_base_dir(project)`, you -just need to switch the version from the `project` back to the previous one. +Check if the change breaks any existing integration you may have that +either runs on the same machine as your repositories are located, or may login to that machine +to access data (for example, a remote backup solution). -```ruby -project.storage_version -# => 2 +To schedule a complete rollout, see the +[rake task documentation for storage migration][rake/migrate-to-hashed] for instructions. -FileUploader.absolute_base_dir(project) -# => "/opt/gitlab/embedded/service/gitlab-rails/public/uploads/@hashed/d4/73/d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35" +If you do have any existing integration, you may want to do a small rollout first, +to validate. You can do so by specifying a range with the operation. -project.storage_version = 1 +This is an example of how to limit the rollout to Project IDs 50 to 100, running in +an Omnibus Gitlab installation: -FileUploader.absolute_base_dir(project) -# => "/opt/gitlab/embedded/service/gitlab-rails/public/uploads/gitlab/gitlab-shell-renamed" +```bash +sudo gitlab-rake gitlab:storage:migrate_to_hashed ID_FROM=50 ID_TO=100 ``` -##### Project +Check the [documentation][rake/migrate-to-hashed] for additional information and instructions for +source-based installation. + +#### Rollback + +Similar to the migration, to disable Hashed Storage for new +projects: -To rollback single Project migration, move `@hashed/aa/bb/aabbcdef1234567890abcdef.git` and `@hashed/aa/bb/aabbcdef1234567890abcdef.wiki.git` -back to `namespace/project.git` and `namespace/project.wiki.git` respectively and switch the version from the `project` back to `null`. +1. Go to **Admin > Settings** and expand the **Repository Storage** section. +2. Uncheck the **Use hashed storage paths for newly created and renamed projects** checkbox. + +To schedule a complete rollback, see the +[rake task documentation for storage rollback][rake/rollback-to-legacy] for instructions. + +The rollback task also supports specifying a range of Project IDs. Here is an example +of limiting the rollout to Project IDs 50 to 100, in an Omnibus Gitlab installation: + +```bash +sudo gitlab-rake gitlab:storage:rollback_to_legacy ID_FROM=50 ID_TO=100 +``` + +If you have a Geo setup, please note that the rollback will not be reflected automatically +on the **secondary** node. You may need to wait for a backfill operation to kick-in and remove +the remaining repositories from the special `@hashed/` folder manually. ### Hashed Storage coverage We are incrementally moving every storable object in GitLab to the Hashed Storage pattern. You can check the current coverage status below (and also see -the [issue](https://gitlab.com/gitlab-com/infrastructure/issues/2821)). +the [issue][ce-2821]). Note that things stored in an S3 compatible endpoint will not have the downsides mentioned earlier, if they are not prefixed with `#{namespace}/#{project_name}`, @@ -156,6 +171,7 @@ which is true for CI Cache and LFS Objects. | CI Artifacts | No | No | Yes | 9.4 / 10.6 | | CI Cache | No | No | Yes | - | | LFS Objects | Yes | Similar | Yes | 10.0 / 10.7 | +| Repository pools| No | Yes | - | 11.6 | #### Implementation Details @@ -180,3 +196,10 @@ LFS Objects implements a similar storage pattern using 2 chars, 2 level folders, ``` They are also S3 compatible since **10.0** (GitLab Premium), and available in GitLab Core since **10.7**. + +[ce-2821]: https://gitlab.com/gitlab-com/infrastructure/issues/2821 +[ce-28283]: https://gitlab.com/gitlab-org/gitlab-ce/issues/28283 +[rake/migrate-to-hashed]: raketasks/storage.md#migrate-existing-projects-to-hashed-storage +[rake/rollback-to-legacy]: raketasks/storage.md#rollback +[storage-paths]: repository_storage_types.md +[gitaly]: gitaly/index.md diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml index 4e708f229cde8cbd40b1fba043ffb2b218a48ec3..ef6d7866e858baba55f016a4f64bbf4ee9dd755f 100644 --- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml @@ -21,20 +21,19 @@ dast: allow_failure: true services: - docker:stable-dind - before_script: + script: - export DAST_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')} - | function dast_run() { docker run \ - --env DAST_TARGET_AVAILABILITY_TIMEOUT \ - --volume "$PWD:/output" \ - --volume /var/run/docker.sock:/var/run/docker.sock \ - -w /output \ - "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION" \ - /analyze -t $DAST_WEBSITE \ - "$@" + --env DAST_TARGET_AVAILABILITY_TIMEOUT \ + --volume "$PWD:/output" \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + -w /output \ + "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION" \ + /analyze -t $DAST_WEBSITE \ + "$@" } - script: - | if [ -n "$DAST_AUTH_URL" ] then diff --git a/spec/controllers/admin/projects_controller_spec.rb b/spec/controllers/admin/projects_controller_spec.rb index 8166657f6747e1c2017210047c3d9076616f46d3..4caf8b46519069d4726b1ecb7b98691179a22488 100644 --- a/spec/controllers/admin/projects_controller_spec.rb +++ b/spec/controllers/admin/projects_controller_spec.rb @@ -43,6 +43,16 @@ describe Admin::ProjectsController do end end + describe 'GET /projects.json' do + render_views + + before do + get :index, format: :json + end + + it { is_expected.to respond_with(:success) } + end + describe 'GET /projects/:id' do render_views diff --git a/spec/controllers/dashboard/projects_controller_spec.rb b/spec/controllers/dashboard/projects_controller_spec.rb index 2975205e09c3f7cf9be1532ea6f6b2b762c822bf..649441f4917a9e0fa70b121e5c8f992a2e5f66e8 100644 --- a/spec/controllers/dashboard/projects_controller_spec.rb +++ b/spec/controllers/dashboard/projects_controller_spec.rb @@ -2,4 +2,30 @@ require 'spec_helper' describe Dashboard::ProjectsController do it_behaves_like 'authenticates sessionless user', :index, :atom + + context 'json requests' do + render_views + + let(:user) { create(:user) } + + before do + sign_in(user) + end + + describe 'GET /projects.json' do + before do + get :index, format: :json + end + + it { is_expected.to respond_with(:success) } + end + + describe 'GET /starred.json' do + before do + get :starred, format: :json + end + + it { is_expected.to respond_with(:success) } + end + end end diff --git a/spec/controllers/explore/projects_controller_spec.rb b/spec/controllers/explore/projects_controller_spec.rb index d57367e931ec798df8156b96f5e101a026f742a7..7e20ddca2494a86beaf7d4cc5a04c56d58197dc3 100644 --- a/spec/controllers/explore/projects_controller_spec.rb +++ b/spec/controllers/explore/projects_controller_spec.rb @@ -1,6 +1,36 @@ require 'spec_helper' describe Explore::ProjectsController do + describe 'GET #index.json' do + render_views + + before do + get :index, format: :json + end + + it { is_expected.to respond_with(:success) } + end + + describe 'GET #trending.json' do + render_views + + before do + get :trending, format: :json + end + + it { is_expected.to respond_with(:success) } + end + + describe 'GET #starred.json' do + render_views + + before do + get :starred, format: :json + end + + it { is_expected.to respond_with(:success) } + end + describe 'GET #trending' do context 'sorting by update date' do let(:project1) { create(:project, :public, updated_at: 3.days.ago) } diff --git a/spec/javascripts/filtered_search/visual_token_value_spec.js b/spec/javascripts/filtered_search/visual_token_value_spec.js index f52dc26a7bbb250b0387f284004f5440c4d7bf0c..14217d460cce23cd9ebe0ada5a51deff1c5cf347 100644 --- a/spec/javascripts/filtered_search/visual_token_value_spec.js +++ b/spec/javascripts/filtered_search/visual_token_value_spec.js @@ -317,7 +317,18 @@ describe('Filtered Search Visual Tokens', () => { it('does not update user token appearance for `none` filter', () => { const { subject, tokenValueContainer, tokenValueElement } = findElements(authorToken); - subject.tokenType = 'none'; + subject.tokenValue = 'none'; + + const { updateUserTokenAppearanceSpy } = setupSpies(subject); + subject.render(tokenValueContainer, tokenValueElement); + + expect(updateUserTokenAppearanceSpy.calls.count()).toBe(0); + }); + + it('does not update user token appearance for `None` filter', () => { + const { subject, tokenValueContainer, tokenValueElement } = findElements(authorToken); + + subject.tokenValue = 'None'; const { updateUserTokenAppearanceSpy } = setupSpies(subject); subject.render(tokenValueContainer, tokenValueElement); @@ -328,7 +339,7 @@ describe('Filtered Search Visual Tokens', () => { it('does not update user token appearance for `any` filter', () => { const { subject, tokenValueContainer, tokenValueElement } = findElements(authorToken); - subject.tokenType = 'any'; + subject.tokenValue = 'any'; const { updateUserTokenAppearanceSpy } = setupSpies(subject); subject.render(tokenValueContainer, tokenValueElement); @@ -336,10 +347,21 @@ describe('Filtered Search Visual Tokens', () => { expect(updateUserTokenAppearanceSpy.calls.count()).toBe(0); }); + it('does not update label token color for `None` filter', () => { + const { subject, tokenValueContainer, tokenValueElement } = findElements(bugLabelToken); + + subject.tokenValue = 'None'; + + const { updateLabelTokenColorSpy } = setupSpies(subject); + subject.render(tokenValueContainer, tokenValueElement); + + expect(updateLabelTokenColorSpy.calls.count()).toBe(0); + }); + it('does not update label token color for `none` filter', () => { const { subject, tokenValueContainer, tokenValueElement } = findElements(bugLabelToken); - subject.tokenType = 'none'; + subject.tokenValue = 'none'; const { updateLabelTokenColorSpy } = setupSpies(subject); subject.render(tokenValueContainer, tokenValueElement); @@ -350,7 +372,7 @@ describe('Filtered Search Visual Tokens', () => { it('does not update label token color for `any` filter', () => { const { subject, tokenValueContainer, tokenValueElement } = findElements(bugLabelToken); - subject.tokenType = 'any'; + subject.tokenValue = 'any'; const { updateLabelTokenColorSpy } = setupSpies(subject); subject.render(tokenValueContainer, tokenValueElement); diff --git a/spec/javascripts/vue_mr_widget/components/states/mr_widget_ready_to_merge_spec.js b/spec/javascripts/vue_mr_widget/components/states/mr_widget_ready_to_merge_spec.js index 6ed654250e68a8202f0ea33ee3b1b08cf450d985..30659ad16f3931264791f621f8fd3daff0b2d737 100644 --- a/spec/javascripts/vue_mr_widget/components/states/mr_widget_ready_to_merge_spec.js +++ b/spec/javascripts/vue_mr_widget/components/states/mr_widget_ready_to_merge_spec.js @@ -377,11 +377,29 @@ describe('ReadyToMerge', () => { }); describe('initiateMergePolling', () => { + beforeEach(() => { + jasmine.clock().install(); + }); + + afterEach(() => { + jasmine.clock().uninstall(); + }); + it('should call simplePoll', () => { const simplePoll = spyOnDependency(ReadyToMerge, 'simplePoll'); vm.initiateMergePolling(); - expect(simplePoll).toHaveBeenCalled(); + expect(simplePoll).toHaveBeenCalledWith(jasmine.any(Function), { timeout: 0 }); + }); + + it('should call handleMergePolling', () => { + spyOn(vm, 'handleMergePolling'); + + vm.initiateMergePolling(); + + jasmine.clock().tick(2000); + + expect(vm.handleMergePolling).toHaveBeenCalled(); }); }); diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb index 7e6dfa30e37086a087419799e460461498d1dbfe..8ba6862392c9deac86ae43fe3481e6f58e469007 100644 --- a/spec/lib/gitlab/git/repository_spec.rb +++ b/spec/lib/gitlab/git/repository_spec.rb @@ -1688,6 +1688,11 @@ describe Gitlab::Git::Repository, :seed_helper do expect(repository.delete_config(*%w[does.not.exist test.foo1 test.foo2])).to be_nil + # Workaround for https://github.com/libgit2/rugged/issues/785: If + # Gitaly changes .gitconfig while Rugged has the file loaded + # Rugged::Repository#each_key will report stale values unless a + # lookup is done first. + expect(repository_rugged.config['test.foo1']).to be_nil config_keys = repository_rugged.config.each_key.to_a expect(config_keys).not_to include('test.foo1') expect(config_keys).not_to include('test.foo2') diff --git a/spec/models/merge_request_diff_spec.rb b/spec/models/merge_request_diff_spec.rb index e530e0302f5b44c40b1cc714b6c6d57d58424afd..53f5307ea0b7ff4e9ce2a3281aac5410ba6a5d13 100644 --- a/spec/models/merge_request_diff_spec.rb +++ b/spec/models/merge_request_diff_spec.rb @@ -1,6 +1,8 @@ require 'spec_helper' describe MergeRequestDiff do + include RepoHelpers + let(:diff_with_commits) { create(:merge_request).merge_request_diff } describe 'validations' do @@ -194,6 +196,25 @@ describe MergeRequestDiff do expect(diff_file).to be_binary expect(diff_file.diff).to eq(mr_diff.compare.diffs(paths: [path]).to_a.first.diff) end + + context 'with diffs that contain a null byte' do + let(:filename) { 'test-null.txt' } + let(:content) { "a" * 10000 + "\x00" } + let(:project) { create(:project, :repository) } + let(:branch) { 'null-data' } + let(:target_branch) { 'master' } + + it 'saves diffs correctly' do + create_file_in_repo(project, target_branch, branch, filename, content) + + mr_diff = create(:merge_request, target_project: project, source_project: project, source_branch: branch, target_branch: target_branch).merge_request_diff + diff_file = mr_diff.merge_request_diff_files.find_by(new_path: filename) + + expect(diff_file).to be_binary + expect(diff_file.diff).to eq(mr_diff.compare.diffs(paths: [filename]).to_a.first.diff) + expect(diff_file.diff).to include(content) + end + end end end diff --git a/spec/models/note_spec.rb b/spec/models/note_spec.rb index 385b8a7959f729cfa769381d97dc54efe7abc41f..eb6f6ff5faf911281bc905f32dc61d7a7f425497 100644 --- a/spec/models/note_spec.rb +++ b/spec/models/note_spec.rb @@ -208,6 +208,24 @@ describe Note do end end + describe "edited?" do + let(:note) { build(:note, updated_by_id: nil, created_at: Time.now, updated_at: Time.now + 5.hours) } + + context "with updated_by" do + it "returns true" do + note.updated_by = build(:user) + + expect(note.edited?).to be_truthy + end + end + + context "without updated_by" do + it "returns false" do + expect(note.edited?).to be_falsy + end + end + end + describe "confidential?" do it "delegates to noteable" do issue_note = build(:note, :on_issue) diff --git a/spec/support/helpers/repo_helpers.rb b/spec/support/helpers/repo_helpers.rb index 3c6956cf5e04658b300a6846664b4b1841f94fe7..4af90f4af79b0cda0d96107e9a45a95ac6b593f2 100644 --- a/spec/support/helpers/repo_helpers.rb +++ b/spec/support/helpers/repo_helpers.rb @@ -115,4 +115,18 @@ eos commits: commits ) end + + def create_file_in_repo( + project, start_branch, branch_name, filename, content, + commit_message: 'Add new content') + Files::CreateService.new( + project, + project.owner, + commit_message: commit_message, + start_branch: start_branch, + branch_name: branch_name, + file_path: filename, + file_content: content + ).execute + end end diff --git a/spec/views/projects/settings/operations/show.html.haml_spec.rb b/spec/views/projects/settings/operations/show.html.haml_spec.rb index 1bca8bba940fc474b6a74688db9085d57e8adb03..6762fe3759bef63b64e0499a161067a3e4ff39e0 100644 --- a/spec/views/projects/settings/operations/show.html.haml_spec.rb +++ b/spec/views/projects/settings/operations/show.html.haml_spec.rb @@ -18,6 +18,7 @@ describe 'projects/settings/operations/show' do allow(view).to receive(:error_tracking_setting) .and_return(error_tracking_setting) allow(view).to receive(:current_user).and_return(user) + allow(view).to receive(:incident_management_available?) { false } end let!(:error_tracking_setting) do