diff --git a/app/models/concerns/cache_markdown_field.rb b/app/models/concerns/cache_markdown_field.rb index f5a70040ee920cef2c3378388ebaef3b48fad812..2eedc1439683d19b58ba2cbed6ee9363435af318 100644 --- a/app/models/concerns/cache_markdown_field.rb +++ b/app/models/concerns/cache_markdown_field.rb @@ -8,6 +8,14 @@ # # Corresponding foo_html, bar_html and baz_html fields should exist. module CacheMarkdownField + extend ActiveSupport::Concern + + # Increment this number every time the renderer changes its output + CACHE_VERSION = 1 + + # changes to these attributes cause the cache to be invalidates + INVALIDATED_BY = %w[author project].freeze + # Knows about the relationship between markdown and html field names, and # stores the rendering contexts for the latter class FieldData @@ -34,34 +42,67 @@ module CacheMarkdownField false end - extend ActiveSupport::Concern + # Returns the default Banzai render context for the cached markdown field. + def banzai_render_context(field) + raise ArgumentError.new("Unknown field: #{field.inspect}") unless + cached_markdown_fields.markdown_fields.include?(field) - included do - cattr_reader :cached_markdown_fields do - FieldData.new - end + # Always include a project key, or Banzai complains + project = self.project if self.respond_to?(:project) + context = cached_markdown_fields[field].merge(project: project) + + # Banzai is less strict about authors, so don't always have an author key + context[:author] = self.author if self.respond_to?(:author) - # Returns the default Banzai render context for the cached markdown field. - def banzai_render_context(field) - raise ArgumentError.new("Unknown field: #{field.inspect}") unless - cached_markdown_fields.markdown_fields.include?(field) + context + end - # Always include a project key, or Banzai complains - project = self.project if self.respond_to?(:project) - context = cached_markdown_fields[field].merge(project: project) + # Update every column in a row if any one is invalidated, as we only store + # one version per row + def refresh_markdown_cache!(do_update: false) + options = { skip_project_check: skip_project_check? } - # Banzai is less strict about authors, so don't always have an author key - context[:author] = self.author if self.respond_to?(:author) + updates = cached_markdown_fields.markdown_fields.map do |markdown_field| + [ + cached_markdown_fields.html_field(markdown_field), + Banzai::Renderer.cacheless_render_field(self, markdown_field, options) + ] + end.to_h + updates['cached_markdown_version'] = CacheMarkdownField::CACHE_VERSION - context - end + updates.each {|html_field, data| write_attribute(html_field, data) } + + update_columns(updates) if persisted? && do_update + end + + def cached_html_up_to_date?(markdown_field) + html_field = cached_markdown_fields.html_field(markdown_field) + + markdown_changed = attribute_changed?(markdown_field) || false + html_changed = attribute_changed?(html_field) || false - # Allow callers to look up the cache field name, rather than hardcoding it - def markdown_cache_field_for(field) - raise ArgumentError.new("Unknown field: #{field}") unless - cached_markdown_fields.markdown_fields.include?(field) + CacheMarkdownField::CACHE_VERSION == cached_markdown_version && + (html_changed || markdown_changed == html_changed) + end + + def invalidated_markdown_cache? + cached_markdown_fields.html_fields.any? {|html_field| attribute_invalidated?(html_field) } + end + + def attribute_invalidated?(attr) + __send__("#{attr}_invalidated?") + end + + def cached_html_for(markdown_field) + raise ArgumentError.new("Unknown field: #{field}") unless + cached_markdown_fields.markdown_fields.include?(markdown_field) + + __send__(cached_markdown_fields.html_field(markdown_field)) + end - cached_markdown_fields.html_field(field) + included do + cattr_reader :cached_markdown_fields do + FieldData.new end # Always exclude _html fields from attributes (including serialization). @@ -70,12 +111,16 @@ module CacheMarkdownField def attributes attrs = attributes_before_markdown_cache + attrs.delete('cached_markdown_version') + cached_markdown_fields.html_fields.each do |field| attrs.delete(field) end attrs end + + before_save :refresh_markdown_cache!, if: :invalidated_markdown_cache? end class_methods do @@ -88,25 +133,15 @@ module CacheMarkdownField cached_markdown_fields[markdown_field] = context html_field = cached_markdown_fields.html_field(markdown_field) - cache_method = "#{markdown_field}_cache_refresh".to_sym invalidation_method = "#{html_field}_invalidated?".to_sym - define_method(cache_method) do - options = { skip_project_check: skip_project_check? } - html = Banzai::Renderer.cacheless_render_field(self, markdown_field, options) - __send__("#{html_field}=", html) - true - end - # The HTML becomes invalid if any dependent fields change. For now, assume # author and project invalidate the cache in all circumstances. define_method(invalidation_method) do changed_fields = changed_attributes.keys - invalidations = changed_fields & [markdown_field.to_s, "author", "project"] - !invalidations.empty? + invalidations = changed_fields & [markdown_field.to_s, *INVALIDATED_BY] + !invalidations.empty? || !cached_html_up_to_date?(markdown_field) end - - before_save cache_method, if: invalidation_method end end end diff --git a/changelogs/unreleased/30672-versioned-markdown-cache.yml b/changelogs/unreleased/30672-versioned-markdown-cache.yml new file mode 100644 index 0000000000000000000000000000000000000000..d8f977b01de1bc32350217c3fe1ea6e309de4f1d --- /dev/null +++ b/changelogs/unreleased/30672-versioned-markdown-cache.yml @@ -0,0 +1,4 @@ +--- +title: Replace rake cache:clear:db with an automatic mechanism +merge_request: 10597 +author: diff --git a/db/migrate/20170410133135_add_version_field_to_markdown_cache.rb b/db/migrate/20170410133135_add_version_field_to_markdown_cache.rb new file mode 100644 index 0000000000000000000000000000000000000000..d9209fe57703352ae56f5eb99452c452ac63311e --- /dev/null +++ b/db/migrate/20170410133135_add_version_field_to_markdown_cache.rb @@ -0,0 +1,25 @@ +class AddVersionFieldToMarkdownCache < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + %i[ + abuse_reports + appearances + application_settings + broadcast_messages + issues + labels + merge_requests + milestones + namespaces + notes + projects + releases + snippets + ].each do |table| + add_column table, :cached_markdown_version, :integer, limit: 4 + end + end +end diff --git a/db/schema.rb b/db/schema.rb index f89e9adb7d69b5c124fae841f2723411ab028a0f..290d969d7de82b72843582cca26027ac89155928 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -24,6 +24,7 @@ ActiveRecord::Schema.define(version: 20170419001229) do t.datetime "created_at" t.datetime "updated_at" t.text "message_html" + t.integer "cached_markdown_version" end create_table "appearances", force: :cascade do |t| @@ -34,6 +35,7 @@ ActiveRecord::Schema.define(version: 20170419001229) do t.datetime "created_at", null: false t.datetime "updated_at", null: false t.text "description_html" + t.integer "cached_markdown_version" end create_table "application_settings", force: :cascade do |t| @@ -116,6 +118,7 @@ ActiveRecord::Schema.define(version: 20170419001229) do t.integer "unique_ips_limit_time_window" t.boolean "unique_ips_limit_enabled", default: false, null: false t.decimal "polling_interval_multiplier", default: 1.0, null: false + t.integer "cached_markdown_version" t.boolean "usage_ping_enabled", default: true, null: false t.string "uuid" end @@ -161,6 +164,7 @@ ActiveRecord::Schema.define(version: 20170419001229) do t.string "color" t.string "font" t.text "message_html" + t.integer "cached_markdown_version" end create_table "chat_names", force: :cascade do |t| @@ -479,6 +483,7 @@ ActiveRecord::Schema.define(version: 20170419001229) do t.integer "time_estimate" t.integer "relative_position" t.datetime "closed_at" + t.integer "cached_markdown_version" end add_index "issues", ["assignee_id"], name: "index_issues_on_assignee_id", using: :btree @@ -543,6 +548,7 @@ ActiveRecord::Schema.define(version: 20170419001229) do t.text "description_html" t.string "type" t.integer "group_id" + t.integer "cached_markdown_version" end add_index "labels", ["group_id", "project_id", "title"], name: "index_labels_on_group_id_and_project_id_and_title", unique: true, using: :btree @@ -663,6 +669,7 @@ ActiveRecord::Schema.define(version: 20170419001229) do t.text "title_html" t.text "description_html" t.integer "time_estimate" + t.integer "cached_markdown_version" end add_index "merge_requests", ["assignee_id"], name: "index_merge_requests_on_assignee_id", using: :btree @@ -700,6 +707,7 @@ ActiveRecord::Schema.define(version: 20170419001229) do t.text "title_html" t.text "description_html" t.date "start_date" + t.integer "cached_markdown_version" end add_index "milestones", ["description"], name: "index_milestones_on_description_trigram", using: :gin, opclasses: {"description"=>"gin_trgm_ops"} @@ -726,6 +734,7 @@ ActiveRecord::Schema.define(version: 20170419001229) do t.integer "parent_id" t.boolean "require_two_factor_authentication", default: false, null: false t.integer "two_factor_grace_period", default: 48, null: false + t.integer "cached_markdown_version" end add_index "namespaces", ["created_at"], name: "index_namespaces_on_created_at", using: :btree @@ -760,6 +769,7 @@ ActiveRecord::Schema.define(version: 20170419001229) do t.integer "resolved_by_id" t.string "discussion_id" t.text "note_html" + t.integer "cached_markdown_version" end add_index "notes", ["author_id"], name: "index_notes_on_author_id", using: :btree @@ -956,6 +966,7 @@ ActiveRecord::Schema.define(version: 20170419001229) do t.integer "auto_cancel_pending_pipelines", default: 0, null: false t.boolean "printing_merge_request_link_enabled", default: true, null: false t.string "import_jid" + t.integer "cached_markdown_version" end add_index "projects", ["ci_id"], name: "index_projects_on_ci_id", using: :btree @@ -1028,6 +1039,7 @@ ActiveRecord::Schema.define(version: 20170419001229) do t.datetime "created_at" t.datetime "updated_at" t.text "description_html" + t.integer "cached_markdown_version" end add_index "releases", ["project_id", "tag"], name: "index_releases_on_project_id_and_tag", using: :btree @@ -1099,6 +1111,7 @@ ActiveRecord::Schema.define(version: 20170419001229) do t.integer "visibility_level", default: 0, null: false t.text "title_html" t.text "content_html" + t.integer "cached_markdown_version" end add_index "snippets", ["author_id"], name: "index_snippets_on_author_id", using: :btree diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index e291bacc53c51392ce56f1168e29576e1ea5ab53..c7801cb5baf7f3280b3b5ad506f673188df63b06 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -33,20 +33,12 @@ module Banzai # of HTML. This method is analogous to calling render(object.field), but it # can cache the rendered HTML in the object, rather than Redis. # - # The context to use is learned from the passed-in object by calling - # #banzai_render_context(field), and cannot be changed. Use #render, passing - # it the field text, if a custom rendering is needed. The generated context - # is returned along with the HTML. + # The context to use is managed by the object and cannot be changed. + # Use #render, passing it the field text, if a custom rendering is needed. def self.render_field(object, field) - html_field = object.markdown_cache_field_for(field) + object.refresh_markdown_cache!(do_update: update_object?(object)) unless object.cached_html_up_to_date?(field) - html = object.__send__(html_field) - return html if html.present? - - html = cacheless_render_field(object, field) - update_object(object, html_field, html) unless object.new_record? || object.destroyed? - - html + object.cached_html_for(field) end # Same as +render_field+, but without consulting or updating the cache field @@ -165,8 +157,9 @@ module Banzai Rails.cache.send(:expanded_key, full_cache_key(cache_key, pipeline_name)) end - def self.update_object(object, html_field, html) - object.update_column(html_field, html) + # GitLab EE needs to disable updates on GET requests in Geo + def self.update_object?(object) + true end end end diff --git a/spec/lib/banzai/object_renderer_spec.rb b/spec/lib/banzai/object_renderer_spec.rb index 4817fcd031a4bc23379a7b927a493408226f1b1b..dd2674f9f207dcfb487040308209993533a59453 100644 --- a/spec/lib/banzai/object_renderer_spec.rb +++ b/spec/lib/banzai/object_renderer_spec.rb @@ -4,13 +4,13 @@ describe Banzai::ObjectRenderer do let(:project) { create(:empty_project) } let(:user) { project.owner } let(:renderer) { described_class.new(project, user, custom_value: 'value') } - let(:object) { Note.new(note: 'hello', note_html: '
hello
') } + let(:object) { Note.new(note: 'hello', note_html: 'hello
', cached_markdown_version: CacheMarkdownField::CACHE_VERSION) } describe '#render' do it 'renders and redacts an Array of objects' do renderer.render([object], :note) - expect(object.redacted_note_html).to eq 'hello
' + expect(object.redacted_note_html).to eq 'hello
' expect(object.user_visible_reference_count).to eq 0 end diff --git a/spec/lib/banzai/renderer_spec.rb b/spec/lib/banzai/renderer_spec.rb index aaa6b12e67ea67d6470a402cb547aab20e97743e..e6f8d2a1fed62bbd0a337e07920e65a8fb9d127e 100644 --- a/spec/lib/banzai/renderer_spec.rb +++ b/spec/lib/banzai/renderer_spec.rb @@ -1,73 +1,36 @@ require 'spec_helper' describe Banzai::Renderer do - def expect_render(project = :project) - expected_context = { project: project } - expect(renderer).to receive(:cacheless_render) { :html }.with(:markdown, expected_context) - end - - def expect_cache_update - expect(object).to receive(:update_column).with("field_html", :html) - end - - def fake_object(*features) - markdown = :markdown if features.include?(:markdown) - html = :html if features.include?(:html) - - object = double( - "object", - banzai_render_context: { project: :project }, - field: markdown, - field_html: html - ) + def fake_object(fresh:) + object = double('object') - allow(object).to receive(:markdown_cache_field_for).with(:field).and_return("field_html") - allow(object).to receive(:new_record?).and_return(features.include?(:new)) - allow(object).to receive(:destroyed?).and_return(features.include?(:destroyed)) + allow(object).to receive(:cached_html_up_to_date?).with(:field).and_return(fresh) + allow(object).to receive(:cached_html_for).with(:field).and_return('field_html') object end - describe "#render_field" do + describe '#render_field' do let(:renderer) { Banzai::Renderer } - let(:subject) { renderer.render_field(object, :field) } + subject { renderer.render_field(object, :field) } - context "with an empty cache" do - let(:object) { fake_object(:markdown) } - it "caches and returns the result" do - expect_render - expect_cache_update - expect(subject).to eq(:html) - end - end + context 'with a stale cache' do + let(:object) { fake_object(fresh: false) } - context "with a filled cache" do - let(:object) { fake_object(:markdown, :html) } + it 'caches and returns the result' do + expect(object).to receive(:refresh_markdown_cache!).with(do_update: true) - it "uses the cache" do - expect_render.never - expect_cache_update.never - should eq(:html) + is_expected.to eq('field_html') end end - context "new object" do - let(:object) { fake_object(:new, :markdown) } - - it "doesn't cache the result" do - expect_render - expect_cache_update.never - expect(subject).to eq(:html) - end - end + context 'with an up-to-date cache' do + let(:object) { fake_object(fresh: true) } - context "destroyed object" do - let(:object) { fake_object(:destroyed, :markdown) } + it 'uses the cache' do + expect(object).to receive(:refresh_markdown_cache!).never - it "doesn't cache the result" do - expect_render - expect_cache_update.never - expect(subject).to eq(:html) + is_expected.to eq('field_html') end end end diff --git a/spec/models/concerns/cache_markdown_field_spec.rb b/spec/models/concerns/cache_markdown_field_spec.rb index 568b34aa42af0401c0729ad3a8023ad29dc12c45..de0069bdcac5756cc1d2dbc3b44f062a20244ef6 100644 --- a/spec/models/concerns/cache_markdown_field_spec.rb +++ b/spec/models/concerns/cache_markdown_field_spec.rb @@ -24,18 +24,19 @@ describe CacheMarkdownField do cache_markdown_field :foo cache_markdown_field :baz, pipeline: :single_line - def self.add_attr(attr_name) - self.attribute_names += [attr_name] - define_attribute_methods(attr_name) - attr_reader(attr_name) - define_method("#{attr_name}=") do |val| - send("#{attr_name}_will_change!") unless val == send(attr_name) - instance_variable_set("@#{attr_name}", val) + def self.add_attr(name) + self.attribute_names += [name] + define_attribute_methods(name) + attr_reader(name) + define_method("#{name}=") do |value| + write_attribute(name, value) end end - [:foo, :foo_html, :bar, :baz, :baz_html].each do |attr_name| - add_attr(attr_name) + add_attr :cached_markdown_version + + [:foo, :foo_html, :bar, :baz, :baz_html].each do |name| + add_attr(name) end def initialize(*) @@ -45,6 +46,15 @@ describe CacheMarkdownField do clear_changes_information end + def read_attribute(name) + instance_variable_get("@#{name}") + end + + def write_attribute(name, value) + send("#{name}_will_change!") unless value == read_attribute(name) + instance_variable_set("@#{name}", value) + end + def save run_callbacks :save do changes_applied @@ -56,115 +66,232 @@ describe CacheMarkdownField do Class.new(ThingWithMarkdownFields) { add_attr(new_attr) } end - let(:markdown) { "`Foo`" } - let(:html) { "Foo
Foo
Bar
Bar