diff --git a/app/models/repository.rb b/app/models/repository.rb index 3ec7d78d5f7b6485a126dfeff05483c4f17e51ff..e620a097d1e6a5dc1972bc0d5a1e0d6b76ac4ff4 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -1127,8 +1127,8 @@ class Repository private - # TODO Generice finder, later split this on finders by Ref or Oid - # https://gitlab.com/gitlab-org/gitlab-foss/issues/39239 + # TODO Genericize finder, later split this on finders by Ref or Oid + # https://gitlab.com/gitlab-org/gitlab/issues/19877 def find_commit(oid_or_ref) commit = if oid_or_ref.is_a?(Gitlab::Git::Commit) oid_or_ref diff --git a/changelogs/unreleased/22166-fix-sast-template.yml b/changelogs/unreleased/22166-fix-sast-template.yml new file mode 100644 index 0000000000000000000000000000000000000000..0c5e17b38ece128ecdff466a9bd8f170c5d6ea93 --- /dev/null +++ b/changelogs/unreleased/22166-fix-sast-template.yml @@ -0,0 +1,5 @@ +--- +title: Check both SAST_DISABLE and SAST_DISABLE_DIND when executing SAST job template +merge_request: 22166 +author: +type: fixed diff --git a/changelogs/unreleased/39951-fix-dependency-scanning-regex.yml b/changelogs/unreleased/39951-fix-dependency-scanning-regex.yml new file mode 100644 index 0000000000000000000000000000000000000000..8866b31753c1040ebf435139deedd41f1af4339a --- /dev/null +++ b/changelogs/unreleased/39951-fix-dependency-scanning-regex.yml @@ -0,0 +1,5 @@ +--- +title: Fix regex matching for gemnasium dependency scanning jobs +merge_request: 22025 +author: Maximilian Stendler +type: fix diff --git a/changelogs/unreleased/remove-ancestor-flag.yml b/changelogs/unreleased/remove-ancestor-flag.yml new file mode 100644 index 0000000000000000000000000000000000000000..2abe15fb72094cbdf42ac5d7dc2e00be29fac388 --- /dev/null +++ b/changelogs/unreleased/remove-ancestor-flag.yml @@ -0,0 +1,5 @@ +--- +title: Remove N+1 query issue when checking group root ancestor. +merge_request: +author: +type: performance diff --git a/db/migrate/20191126134210_rename_packages_package_tags.rb b/db/migrate/20191126134210_rename_packages_package_tags.rb new file mode 100644 index 0000000000000000000000000000000000000000..75cb53802abd0c1472591c265a41cfa2a12282cc --- /dev/null +++ b/db/migrate/20191126134210_rename_packages_package_tags.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class RenamePackagesPackageTags < ActiveRecord::Migration[5.2] + DOWNTIME = false + + def change + rename_table(:packages_package_tags, :packages_tags) + end +end diff --git a/db/migrate/20200106085831_add_timestamps_to_packages_tags.rb b/db/migrate/20200106085831_add_timestamps_to_packages_tags.rb new file mode 100644 index 0000000000000000000000000000000000000000..2720d9b3297ff4f91f7ef129b1132c6d1b0643e3 --- /dev/null +++ b/db/migrate/20200106085831_add_timestamps_to_packages_tags.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class AddTimestampsToPackagesTags < ActiveRecord::Migration[5.2] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + # We disable these cops here because adding this column is safe. The table does not + # have any data in it. + # rubocop: disable Migration/AddIndex + def up + add_timestamps_with_timezone(:packages_tags, null: false) + add_index(:packages_tags, [:package_id, :updated_at], order: { updated_at: :desc }) + end + + # We disable these cops here because adding this column is safe. The table does not + # have any data in it. + # rubocop: disable Migration/RemoveIndex + def down + remove_index(:packages_tags, [:package_id, :updated_at]) + remove_timestamps(:packages_tags) + end +end diff --git a/db/schema.rb b/db/schema.rb index 3ead1f38ac35cd9afd88042fa7e92f449c62a368..2b6f847441214e7b9bf3e58dabba6125b5b5a44c 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2020_01_06_071113) do +ActiveRecord::Schema.define(version: 2020_01_06_085831) do # These are extensions that must be enabled in order to support this database enable_extension "pg_trgm" @@ -2938,12 +2938,6 @@ ActiveRecord::Schema.define(version: 2020_01_06_071113) do t.index ["package_id", "file_name"], name: "index_packages_package_files_on_package_id_and_file_name" end - create_table "packages_package_tags", force: :cascade do |t| - t.integer "package_id", null: false - t.string "name", limit: 255, null: false - t.index ["package_id"], name: "index_packages_package_tags_on_package_id" - end - create_table "packages_packages", force: :cascade do |t| t.integer "project_id", null: false t.datetime_with_timezone "created_at", null: false @@ -2956,6 +2950,15 @@ ActiveRecord::Schema.define(version: 2020_01_06_071113) do t.index ["project_id"], name: "index_packages_packages_on_project_id" end + create_table "packages_tags", force: :cascade do |t| + t.integer "package_id", null: false + t.string "name", limit: 255, null: false + t.datetime_with_timezone "created_at", null: false + t.datetime_with_timezone "updated_at", null: false + t.index ["package_id", "updated_at"], name: "index_packages_tags_on_package_id_and_updated_at", order: { updated_at: :desc } + t.index ["package_id"], name: "index_packages_tags_on_package_id" + end + create_table "pages_domain_acme_orders", force: :cascade do |t| t.integer "pages_domain_id", null: false t.datetime_with_timezone "expires_at", null: false @@ -4705,8 +4708,8 @@ ActiveRecord::Schema.define(version: 2020_01_06_071113) do add_foreign_key "packages_dependency_links", "packages_packages", column: "package_id", on_delete: :cascade add_foreign_key "packages_maven_metadata", "packages_packages", column: "package_id", name: "fk_be88aed360", on_delete: :cascade add_foreign_key "packages_package_files", "packages_packages", column: "package_id", name: "fk_86f0f182f8", on_delete: :cascade - add_foreign_key "packages_package_tags", "packages_packages", column: "package_id", on_delete: :cascade add_foreign_key "packages_packages", "projects", on_delete: :cascade + add_foreign_key "packages_tags", "packages_packages", column: "package_id", on_delete: :cascade add_foreign_key "pages_domain_acme_orders", "pages_domains", on_delete: :cascade add_foreign_key "pages_domains", "projects", name: "fk_ea2f6dfc6f", on_delete: :cascade add_foreign_key "path_locks", "projects", name: "fk_5265c98f24", on_delete: :cascade diff --git a/doc/administration/gitaly/index.md b/doc/administration/gitaly/index.md index c525a069a9e7cea204175247cd9a7091889bb95d..f762c0417f443b1c7a33d9fd578fb4fbc35199a5 100644 --- a/doc/administration/gitaly/index.md +++ b/doc/administration/gitaly/index.md @@ -164,11 +164,21 @@ Git operations in GitLab will result in an API error. postgresql['enable'] = false redis['enable'] = false nginx['enable'] = false - prometheus['enable'] = false unicorn['enable'] = false sidekiq['enable'] = false gitlab_workhorse['enable'] = false + # If you don't want to run monitoring services uncomment the following (not recommended) + # alertmanager['enable'] = false + # gitlab_exporter['enable'] = false + # grafana['enable'] = false + # node_exporter['enable'] = false + # prometheus['enable'] = false + + # Enable prometheus monitoring - comment out if you disable monitoring services above. + # This makes Prometheus listen on all interfaces. You must use firewalls to restrict access to this address/port. + prometheus['listen_address'] = '0.0.0.0:9090' + # Prevent database connections during 'gitlab-ctl reconfigure' gitlab_rails['rake_cache_clear'] = false gitlab_rails['auto_migrate'] = false @@ -191,6 +201,11 @@ Git operations in GitLab will result in an API error. 1. Append the following to `/etc/gitlab/gitlab.rb` for each respective server: + + On `gitaly1.internal`: ``` @@ -561,14 +576,14 @@ a few things that you need to do: 1. Make sure the [`git` user home directory](https://docs.gitlab.com/omnibus/settings/configuration.html#moving-the-home-directory-for-a-user) is on local disk. 1. Configure [database lookup of SSH keys](../operations/fast_ssh_key_lookup.md) - to eliminate the need for a shared authorized_keys file. + to eliminate the need for a shared `authorized_keys` file. 1. Configure [object storage for job artifacts](../job_artifacts.md#using-object-storage) including [incremental logging](../job_logs.md#new-incremental-logging-architecture). 1. Configure [object storage for LFS objects](../lfs/lfs_administration.md#storing-lfs-objects-in-remote-object-storage). 1. Configure [object storage for uploads](../uploads.md#using-object-storage-core-only). -1. Configure [object storage for Merge Request Diffs](../merge_request_diffs.md#using-object-storage). -1. Configure [object storage for Packages](../packages/index.md#using-object-storage) (Optional Feature). -1. Configure [object storage for Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (Optional Feature). +1. Configure [object storage for merge request diffs](../merge_request_diffs.md#using-object-storage). +1. Configure [object storage for packages](../packages/index.md#using-object-storage) (optional feature). +1. Configure [object storage for dependency proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature). NOTE: **Note:** One current feature of GitLab that still requires a shared directory (NFS) is @@ -862,7 +877,7 @@ remote: GitLab: 401 Unauthorized You will need to sync your `gitlab-secrets.json` file with your GitLab app nodes. -### Client side GRPC logs +### Client side gRPC logs Gitaly uses the [gRPC](https://grpc.io/) RPC framework. The Ruby gRPC client has its own log file which may contain useful information when diff --git a/doc/administration/packages/container_registry.md b/doc/administration/packages/container_registry.md index e735d8dd97eda741d180df4dfde0c99f6c37d6fc..2e97849eda23806b3569bf7ec8a9f7d2d7c109ce 100644 --- a/doc/administration/packages/container_registry.md +++ b/doc/administration/packages/container_registry.md @@ -632,6 +632,29 @@ mounting the docker-daemon and setting `privileged = false` in the Runner's Additional information about this: [issue 18239](https://gitlab.com/gitlab-org/gitlab-foss/issues/18239). +### `unauthorized: authentication required` when pushing large images + +Example error: + +```shell +docker push gitlab.example.com/myproject/docs:latest +The push refers to a repository [gitlab.example.com/myproject/docs] +630816f32edb: Preparing +530d5553aec8: Preparing +... +4b0bab9ff599: Waiting +d1c800db26c7: Waiting +42755cf4ee95: Waiting +unauthorized: authentication required +``` + +GitLab has a default token expiration of 5 minutes for the registry. When pushing +larger images, or images that take longer than 5 minutes to push, users may +encounter this error. + +Administrators can increase the token duration in **Admin area > Settings > +Container Registry > Authorization token duration (minutes)**. + ### AWS S3 with the GitLab registry error when pushing large images When using AWS S3 with the GitLab registry, an error may occur when pushing diff --git a/doc/development/code_review.md b/doc/development/code_review.md index c21b9912e8f813603b3d48d50f6a2a40fe8ae260..445feba384b059d65e718121a482181aca3789c9 100644 --- a/doc/development/code_review.md +++ b/doc/development/code_review.md @@ -274,6 +274,12 @@ time frame, let the author know as soon as possible and try to help them find another reviewer or maintainer who will be able to, so that they can be unblocked and get on with their work quickly. +If you think you are at capacity and are unable to accept any more reviews until +some have been completed, communicate this through your GitLab status by setting +the `:red_circle:` emoji and mentioning that you are at capacity in the status +text. This will guide contributors to pick a different reviewer, helping us to +meet the SLO. + Of course, if you are out of office and have [communicated](https://about.gitlab.com/handbook/paid-time-off/#communicating-your-time-off) this through your GitLab.com Status, authors are expected to realize this and diff --git a/doc/integration/elasticsearch.md b/doc/integration/elasticsearch.md index 292cddd284be2e54891521a3f013dc84e86336cb..2d827f53fcf3c220e9bc11bc57e174837fa2a6bf 100644 --- a/doc/integration/elasticsearch.md +++ b/doc/integration/elasticsearch.md @@ -36,6 +36,9 @@ it yourself or by using the service. Running Elasticsearch on the same server as GitLab is not recommended and it will likely cause performance degradation on the GitLab installation. +NOTE: **Note:** +**For a single node Elasticsearch cluster the functional cluster health status will be yellow** (will never be green) because the primary shard is allocated but replicas can not be as there is no other node to which Elasticsearch can assign a replica. + Once the data is added to the database or repository and [Elasticsearch is enabled in the admin area](#enabling-elasticsearch) the search index will be updated automatically. @@ -591,6 +594,23 @@ Here are some common pitfalls and how to overcome them: AWS has [fixed limits](https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/aes-limits.html) for this setting ("Maximum Size of HTTP Request Payloads"), based on the size of the underlying instance. + +- **My single node Elasticsearch cluster status never goes from `yellow` to `green` even though everything seems to be running properly** + + **For a single node Elasticsearch cluster the functional cluster health status will be yellow** (will never be green) because the primary shard is allocated but replicas can not be as there is no other node to which Elasticsearch can assign a replica. This also applies if you are using using the +[Amazon Elasticsearch](https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/aes-handling-errors.html#aes-handling-errors-yellow-cluster-status) service. + + CAUTION: **Warning**: Setting the number of replicas to `0` is not something that we recommend (this is not allowed in the GitLab Elasticsearch Integration menu). If you are planning to add more Elasticsearch nodes (for a total of more than 1 Elasticsearch) the number of replicas will need to be set to an integer value larger than `0`. Failure to do so will result in lack of redundancy (losing one node will corupt the index). + + If you have a **hard requirement to have a green status for your single node Elasticsearch cluster**, please make sure you understand the risks outlined in the previous paragraph and then simply run the following query to set the number of replicas to `0`(the cluster will no longer try to create any shard replicas): + + ```bash + curl --request PUT localhost:9200/gitlab-production/_settings --header 'Content-Type: application/json' --data '{ + "index" : { + "number_of_replicas" : 0 + } + }' + ``` ### Reverting to basic search diff --git a/doc/security/webhooks.md b/doc/security/webhooks.md index cb9ad2b694c91f60b9d216bc83af4ea48f0ba63e..d7baf78690687d936bcfa10a32a40a48c56a6ac1 100644 --- a/doc/security/webhooks.md +++ b/doc/security/webhooks.md @@ -35,8 +35,8 @@ to endpoints like `http://localhost:123/some-resource/delete`. To prevent this type of exploitation from happening, starting with GitLab 10.6, all Webhook requests to the current GitLab instance server address and/or in a private network will be forbidden by default. That means that all requests made -to 127.0.0.1, ::1 and 0.0.0.0, as well as IPv4 10.0.0.0/8, 172.16.0.0/12, -192.168.0.0/16 and IPv6 site-local (ffc0::/10) addresses won't be allowed. +to `127.0.0.1`, `::1` and `0.0.0.0`, as well as IPv4 `10.0.0.0/8`, `172.16.0.0/12`, +`192.168.0.0/16` and IPv6 site-local (`ffc0::/10`) addresses won't be allowed. This behavior can be overridden by enabling the option *"Allow requests to the local network from web hooks and services"* in the *"Outbound requests"* section diff --git a/doc/user/packages/dependency_proxy/img/group_dependency_proxy.png b/doc/user/packages/dependency_proxy/img/group_dependency_proxy.png index 035aff0b6c47cec54203b77c8f52fef9b7f10261..42b412911f0677e14e94dd09d25d8c19237d0f7c 100644 Binary files a/doc/user/packages/dependency_proxy/img/group_dependency_proxy.png and b/doc/user/packages/dependency_proxy/img/group_dependency_proxy.png differ diff --git a/doc/user/packages/dependency_proxy/index.md b/doc/user/packages/dependency_proxy/index.md index 60f4dbc0abba03a2da3c8bb54a67384f2aeb0130..05934212a122070f8fdeb51ac5a48dcf762d4b00 100644 --- a/doc/user/packages/dependency_proxy/index.md +++ b/doc/user/packages/dependency_proxy/index.md @@ -12,7 +12,7 @@ receiving a request and returning the upstream image from a registry, acting as a pull-through cache. The dependency proxy is available in the group level. To access it, navigate to -a group's **Overview > Dependency Proxy**. +a group's **Packages > Dependency Proxy**. ![Dependency Proxy group page](img/group_dependency_proxy.png) @@ -33,7 +33,7 @@ The following dependency proxies are supported. With the Docker dependency proxy, you can use GitLab as a source for a Docker image. To get a Docker image into the dependency proxy: -1. Find the proxy URL on your group's page under **Overview > Dependency Proxy**, +1. Find the proxy URL on your group's page under **Packages > Dependency Proxy**, for example `gitlab.com/groupname/dependency_proxy/containers`. 1. Trigger GitLab to pull the Docker image you want (e.g., `alpine:latest` or `linuxserver/nextcloud:latest`) and store it in the proxy storage by using diff --git a/doc/user/packages/npm_registry/index.md b/doc/user/packages/npm_registry/index.md index 7d5db5a60ef1b53a3668129d18b6c47f7ae1a987..1ac1dcc18972814cb6901323e25ff96da121b70d 100644 --- a/doc/user/packages/npm_registry/index.md +++ b/doc/user/packages/npm_registry/index.md @@ -134,8 +134,8 @@ Add a corresponding section to your `.npmrc` file: ```ini @foo:registry=https://gitlab.com/api/v4/packages/npm/ -//gitlab.com/api/v4/packages/npm/:_authToken=${env.CI_JOB_TOKEN} -//gitlab.com/api/v4/projects/{env.CI_PROJECT_ID>/packages/npm/:_authToken=${env.CI_JOB_TOKEN} +//gitlab.com/api/v4/packages/npm/:_authToken=${CI_JOB_TOKEN} +//gitlab.com/api/v4/projects/${CI_PROJECT_ID}/packages/npm/:_authToken=${CI_JOB_TOKEN} ``` ## Uploading packages @@ -242,3 +242,27 @@ Starting from GitLab 12.6, new packages published to the GitLab NPM Registry exp - bundleDependencies - peerDependencies - deprecated + +## NPM distribution tags + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/9425) in GitLab Premium 12.7. + +Dist Tags for newly published packages are supported, and they follow NPM's convention where they are optional, and each tag can only be assigned to 1 package at +You can add [distribution tags](https://docs.npmjs.com/cli/dist-tag) for newly +published packages. They follow NPM's convention where they are optional, and +each tag can only be assigned to one package at a time. The latest tag is added +by default when a package is published without a tag. The same goes to installing +a package without specifying the tag or version. + +Examples of the supported `dist-tag` commands and using tags in general: + +```sh +npm publish @scope/package --tag # Publish new package with new tag +npm dist-tag add @scope/package@version my-tag # Add a tag to an existing package +npm dist-tag ls @scope/package # List all tags under the package +npm dist-tag rm @scope/package@version my-tag # Delete a tag from the package +npm install @scope/package@my-tag # Install a specific tag +``` + +CAUTION: **Warning:** +Due to a bug in NPM 6.9.0, deleting dist tags fails. Make sure your NPM version is greater than 6.9.1. diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index d73f6ccdb3f042e6719c349c84ed069c8394abe3..e979a6f2de1c20d705422fd2545e456a16d625ce 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -88,7 +88,7 @@ gemnasium-dependency_scanning: only: variables: - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && - $DS_DEFAULT_ANALYZERS =~ /gemnasium/ && + $DS_DEFAULT_ANALYZERS =~ /gemnasium([^-]|$)/ && $CI_PROJECT_REPOSITORY_LANGUAGES =~ /ruby|javascript|php/ gemnasium-maven-dependency_scanning: diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 34d84138a8bd9c64579868cc95da474e0fa6b66a..864e3eb569d4e2120a0bf7b93368d6e9b122db2a 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -55,6 +55,7 @@ sast: services: [] except: variables: + - $SAST_DISABLED - $SAST_DISABLE_DIND == 'false' script: - /analyzer run