From: Tom Hughes Date: Tue, 2 Jul 2019 19:47:42 +0000 (+0100) Subject: Add support for Active Storage attachments X-Git-Tag: live~2466^2~4 X-Git-Url: https://git.openstreetmap.org/rails.git/commitdiff_plain/ba627420a3e183fcee7ce38b2f68a0ff7663c647 Add support for Active Storage attachments --- diff --git a/.gitignore b/.gitignore index 386f0513a..1328658c6 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,5 @@ node_modules public/assets public/attachments public/export +storage tmp diff --git a/.travis.yml b/.travis.yml index 4d764d6ed..e4c81e4c1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,6 +25,7 @@ before_script: - psql -U postgres -c "CREATE FUNCTION tile_for_point(int4, int4) RETURNS int8 AS '/tmp/libpgosm', 'tile_for_point' LANGUAGE C STRICT" openstreetmap - psql -U postgres -c "CREATE FUNCTION xid_to_int4(xid) RETURNS int4 AS '/tmp/libpgosm', 'xid_to_int4' LANGUAGE C STRICT" openstreetmap - cp config/travis.database.yml config/database.yml + - cp config/example.storage.yml config/storage.yml - touch config/settings.local.yml - bundle exec rake db:migrate - bundle exec rake i18n:js:export diff --git a/INSTALL.md b/INSTALL.md index fafeb0000..d2d2b79b4 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -124,6 +124,15 @@ We use [Yarn](https://yarnpkg.com/) to manage the Node.js modules required for t bundle exec rake yarn:install ``` +## Storage setup + +The Rails port needs to be configured with an object storage facility - for +development and testing purposes you can use the example configuration: + +``` +cp config/example.storage.yml config/storage.yml +``` + ## Database setup The Rails Port uses three databases - one for development, one for testing, and one for production. The database-specific configuration diff --git a/config/.gitignore b/config/.gitignore index b5649dd03..95ba2dbdb 100644 --- a/config/.gitignore +++ b/config/.gitignore @@ -1 +1,2 @@ database.yml +storage.yml diff --git a/config/environments/production.rb b/config/environments/production.rb index 97121d5a7..90c0c1c40 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -39,7 +39,7 @@ Rails.application.configure do # config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for NGINX # Store uploaded files on the local file system (see config/storage.yml for options) - config.active_storage.service = :local + config.active_storage.service = Settings.storage_service.to_symbol # Mount Action Cable outside main process or domain # config.action_cable.mount_path = nil diff --git a/config/example.storage.yml b/config/example.storage.yml new file mode 100644 index 000000000..695f17bd3 --- /dev/null +++ b/config/example.storage.yml @@ -0,0 +1,7 @@ +test: + service: Disk + root: <%= Rails.root.join("tmp/storage") %> + +local: + service: Disk + root: <%= Rails.root.join("storage") %> diff --git a/config/initializers/config.rb b/config/initializers/config.rb index f7d5d7924..271bd79e7 100644 --- a/config/initializers/config.rb +++ b/config/initializers/config.rb @@ -77,5 +77,6 @@ Config.setup do |config| required(:api_timeout).filled(:int?) required(:imagery_blacklist).maybe(:array?) required(:status).filled(:str?, :included_in? => ALLOWED_STATUS) + required(:storage_service).filled(:str?) end end diff --git a/config/settings.yml b/config/settings.yml index a664b78b0..d91f52f08 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -122,3 +122,5 @@ fossgis_osrm_url: "https://routing.openstreetmap.de/" csp_enforce: false # URL for reporting Content-Security-Policy violations #csp_report_url: "" +# Storage service to use in production mode +storage_service: "local" diff --git a/config/storage.yml b/config/storage.yml deleted file mode 100644 index d32f76e8f..000000000 --- a/config/storage.yml +++ /dev/null @@ -1,34 +0,0 @@ -test: - service: Disk - root: <%= Rails.root.join("tmp/storage") %> - -local: - service: Disk - root: <%= Rails.root.join("storage") %> - -# Use rails credentials:edit to set the AWS secrets (as aws:access_key_id|secret_access_key) -# amazon: -# service: S3 -# access_key_id: <%= Rails.application.credentials.dig(:aws, :access_key_id) %> -# secret_access_key: <%= Rails.application.credentials.dig(:aws, :secret_access_key) %> -# region: us-east-1 -# bucket: your_own_bucket - -# Remember not to checkin your GCS keyfile to a repository -# google: -# service: GCS -# project: your_project -# credentials: <%= Rails.root.join("path/to/gcs.keyfile") %> -# bucket: your_own_bucket - -# Use rails credentials:edit to set the Azure Storage secret (as azure_storage:storage_access_key) -# microsoft: -# service: AzureStorage -# storage_account_name: your_account_name -# storage_access_key: <%= Rails.application.credentials.dig(:azure_storage, :storage_access_key) %> -# container: your_container_name - -# mirror: -# service: Mirror -# primary: local -# mirrors: [ amazon, google, microsoft ] diff --git a/db/migrate/20190702193519_create_active_storage_tables.rb b/db/migrate/20190702193519_create_active_storage_tables.rb new file mode 100644 index 000000000..1968b7b8a --- /dev/null +++ b/db/migrate/20190702193519_create_active_storage_tables.rb @@ -0,0 +1,27 @@ +# This migration comes from active_storage (originally 20170806125915) +class CreateActiveStorageTables < ActiveRecord::Migration[5.2] + def change + create_table :active_storage_blobs do |t| + t.string :key, :null => false + t.string :filename, :null => false + t.string :content_type + t.text :metadata + t.bigint :byte_size, :null => false + t.string :checksum, :null => false + t.datetime :created_at, :null => false + + t.index [:key], :unique => true + end + + create_table :active_storage_attachments do |t| + t.string :name, :null => false + t.references :record, :null => false, :polymorphic => true, :index => false + t.references :blob, :null => false + + t.datetime :created_at, :null => false + + t.index [:record_type, :record_id, :name, :blob_id], :name => "index_active_storage_attachments_uniqueness", :unique => true + t.foreign_key :active_storage_blobs, :column => :blob_id + end + end +end diff --git a/db/structure.sql b/db/structure.sql index d115140d1..8f4fe3e79 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -192,6 +192,74 @@ CREATE SEQUENCE public.acls_id_seq ALTER SEQUENCE public.acls_id_seq OWNED BY public.acls.id; +-- +-- Name: active_storage_attachments; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.active_storage_attachments ( + id bigint NOT NULL, + name character varying NOT NULL, + record_type character varying NOT NULL, + record_id bigint NOT NULL, + blob_id bigint NOT NULL, + created_at timestamp without time zone NOT NULL +); + + +-- +-- Name: active_storage_attachments_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.active_storage_attachments_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: active_storage_attachments_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.active_storage_attachments_id_seq OWNED BY public.active_storage_attachments.id; + + +-- +-- Name: active_storage_blobs; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.active_storage_blobs ( + id bigint NOT NULL, + key character varying NOT NULL, + filename character varying NOT NULL, + content_type character varying, + metadata text, + byte_size bigint NOT NULL, + checksum character varying NOT NULL, + created_at timestamp without time zone NOT NULL +); + + +-- +-- Name: active_storage_blobs_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.active_storage_blobs_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: active_storage_blobs_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.active_storage_blobs_id_seq OWNED BY public.active_storage_blobs.id; + + -- -- Name: ar_internal_metadata; Type: TABLE; Schema: public; Owner: - -- @@ -1363,6 +1431,20 @@ CREATE TABLE public.ways ( ALTER TABLE ONLY public.acls ALTER COLUMN id SET DEFAULT nextval('public.acls_id_seq'::regclass); +-- +-- Name: active_storage_attachments id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.active_storage_attachments ALTER COLUMN id SET DEFAULT nextval('public.active_storage_attachments_id_seq'::regclass); + + +-- +-- Name: active_storage_blobs id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.active_storage_blobs ALTER COLUMN id SET DEFAULT nextval('public.active_storage_blobs_id_seq'::regclass); + + -- -- Name: changeset_comments id; Type: DEFAULT; Schema: public; Owner: - -- @@ -1546,6 +1628,22 @@ ALTER TABLE ONLY public.acls ADD CONSTRAINT acls_pkey PRIMARY KEY (id); +-- +-- Name: active_storage_attachments active_storage_attachments_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.active_storage_attachments + ADD CONSTRAINT active_storage_attachments_pkey PRIMARY KEY (id); + + +-- +-- Name: active_storage_blobs active_storage_blobs_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.active_storage_blobs + ADD CONSTRAINT active_storage_blobs_pkey PRIMARY KEY (id); + + -- -- Name: ar_internal_metadata ar_internal_metadata_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -2086,6 +2184,27 @@ CREATE INDEX index_acls_on_domain ON public.acls USING btree (domain); CREATE INDEX index_acls_on_mx ON public.acls USING btree (mx); +-- +-- Name: index_active_storage_attachments_on_blob_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_active_storage_attachments_on_blob_id ON public.active_storage_attachments USING btree (blob_id); + + +-- +-- Name: index_active_storage_attachments_uniqueness; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX index_active_storage_attachments_uniqueness ON public.active_storage_attachments USING btree (record_type, record_id, name, blob_id); + + +-- +-- Name: index_active_storage_blobs_on_key; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX index_active_storage_blobs_on_key ON public.active_storage_blobs USING btree (key); + + -- -- Name: index_changeset_comments_on_created_at; Type: INDEX; Schema: public; Owner: - -- @@ -2598,6 +2717,14 @@ ALTER TABLE ONLY public.diary_entry_subscriptions ADD CONSTRAINT diary_entry_subscriptions_user_id_fkey FOREIGN KEY (user_id) REFERENCES public.users(id); +-- +-- Name: active_storage_attachments fk_rails_c3b3935057; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.active_storage_attachments + ADD CONSTRAINT fk_rails_c3b3935057 FOREIGN KEY (blob_id) REFERENCES public.active_storage_blobs(id); + + -- -- Name: friends friends_friend_user_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- @@ -2958,6 +3085,7 @@ INSERT INTO "schema_migrations" (version) VALUES ('20181031113522'), ('20190518115041'), ('20190623093642'), +('20190702193519'), ('21'), ('22'), ('23'),