From 801522c5c349cee50c917a933bcd39656c8dc230 Mon Sep 17 00:00:00 2001 From: Andy Allan Date: Wed, 31 Oct 2018 15:31:32 +0100 Subject: [PATCH] Set up Delayed Job as the backend for Active Job This persists jobs into the database, and uses locking to ensure that workers from multiple machines avoid treading on each other. Jobs can be run by using `bundle exec rake jobs:work` Fixes #2015 --- Gemfile | 1 + Gemfile.lock | 6 ++ config/application.rb | 3 + .../20181031113522_create_delayed_jobs.rb | 22 +++++++ db/structure.sql | 62 +++++++++++++++++++ 5 files changed, 94 insertions(+) create mode 100644 db/migrate/20181031113522_create_delayed_jobs.rb diff --git a/Gemfile b/Gemfile index 9ba270313..d60f1ba94 100644 --- a/Gemfile +++ b/Gemfile @@ -46,6 +46,7 @@ gem "image_optim_rails" # Load rails plugins gem "actionpack-page_caching" gem "composite_primary_keys", "~> 11.0.0" +gem "delayed_job_active_record" gem "dynamic_form" gem "http_accept_language", "~> 2.0.0" gem "i18n-js", ">= 3.0.0" diff --git a/Gemfile.lock b/Gemfile.lock index 76a31e169..9080f009e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -99,6 +99,11 @@ GEM crass (1.0.4) dalli (2.7.8) debug_inspector (0.0.3) + delayed_job (4.1.5) + activesupport (>= 3.0, < 5.3) + delayed_job_active_record (4.1.3) + activerecord (>= 3.0, < 5.3) + delayed_job (>= 3.0, < 5) docile (1.3.1) dynamic_form (1.1.4) erubi (1.7.1) @@ -393,6 +398,7 @@ DEPENDENCIES composite_primary_keys (~> 11.0.0) coveralls dalli + delayed_job_active_record dynamic_form factory_bot_rails faraday diff --git a/config/application.rb b/config/application.rb index 5139111cc..a5431f09f 100644 --- a/config/application.rb +++ b/config/application.rb @@ -51,5 +51,8 @@ module OpenStreetMap config.logstasher.logger_path = LOGSTASH_PATH config.logstasher.log_controller_parameters = true end + + # Use DelayedJob as the persistent backend for ActiveJob + config.active_job.queue_adapter = :delayed_job end end diff --git a/db/migrate/20181031113522_create_delayed_jobs.rb b/db/migrate/20181031113522_create_delayed_jobs.rb new file mode 100644 index 000000000..69c6f1b04 --- /dev/null +++ b/db/migrate/20181031113522_create_delayed_jobs.rb @@ -0,0 +1,22 @@ +class CreateDelayedJobs < ActiveRecord::Migration[5.2] + def self.up + create_table :delayed_jobs, :force => true do |table| + table.integer :priority, :default => 0, :null => false # Allows some jobs to jump to the front of the queue + table.integer :attempts, :default => 0, :null => false # Provides for retries, but still fail eventually. + table.text :handler, :null => false # YAML-encoded string of the object that will do work + table.text :last_error # reason for last failure (See Note below) + table.datetime :run_at # When to run. Could be Time.zone.now for immediately, or sometime in the future. + table.datetime :locked_at # Set when a client is working on this object + table.datetime :failed_at # Set when all retries have failed (actually, by default, the record is deleted instead) + table.string :locked_by # Who is working on this object (if locked) + table.string :queue # The name of the queue this job is in + table.timestamps :null => true + end + + add_index :delayed_jobs, [:priority, :run_at], :name => "delayed_jobs_priority" + end + + def self.down + drop_table :delayed_jobs + end +end diff --git a/db/structure.sql b/db/structure.sql index 48fedeba5..dcab302bb 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -494,6 +494,45 @@ CREATE SEQUENCE current_ways_id_seq ALTER SEQUENCE current_ways_id_seq OWNED BY current_ways.id; +-- +-- Name: delayed_jobs; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE delayed_jobs ( + id bigint NOT NULL, + priority integer DEFAULT 0 NOT NULL, + attempts integer DEFAULT 0 NOT NULL, + handler text NOT NULL, + last_error text, + run_at timestamp without time zone, + locked_at timestamp without time zone, + failed_at timestamp without time zone, + locked_by character varying, + queue character varying, + created_at timestamp without time zone, + updated_at timestamp without time zone +); + + +-- +-- Name: delayed_jobs_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE delayed_jobs_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: delayed_jobs_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE delayed_jobs_id_seq OWNED BY delayed_jobs.id; + + -- -- Name: diary_comments; Type: TABLE; Schema: public; Owner: - -- @@ -1365,6 +1404,13 @@ ALTER TABLE ONLY current_relations ALTER COLUMN id SET DEFAULT nextval('current_ ALTER TABLE ONLY current_ways ALTER COLUMN id SET DEFAULT nextval('current_ways_id_seq'::regclass); +-- +-- Name: delayed_jobs id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY delayed_jobs ALTER COLUMN id SET DEFAULT nextval('delayed_jobs_id_seq'::regclass); + + -- -- Name: diary_comments id; Type: DEFAULT; Schema: public; Owner: - -- @@ -1595,6 +1641,14 @@ ALTER TABLE ONLY current_ways ADD CONSTRAINT current_ways_pkey PRIMARY KEY (id); +-- +-- Name: delayed_jobs delayed_jobs_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY delayed_jobs + ADD CONSTRAINT delayed_jobs_pkey PRIMARY KEY (id); + + -- -- Name: diary_comments diary_comments_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -1918,6 +1972,13 @@ CREATE INDEX current_way_nodes_node_idx ON current_way_nodes USING btree (node_i CREATE INDEX current_ways_timestamp_idx ON current_ways USING btree ("timestamp"); +-- +-- Name: delayed_jobs_priority; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX delayed_jobs_priority ON delayed_jobs USING btree (priority, run_at); + + -- -- Name: diary_comment_user_id_created_at_index; Type: INDEX; Schema: public; Owner: - -- @@ -2870,6 +2931,7 @@ INSERT INTO "schema_migrations" (version) VALUES ('20161011010929'), ('20170222134109'), ('20180204153242'), +('20181031113522'), ('21'), ('22'), ('23'), -- 2.43.2