From: Tom Hughes Date: Wed, 16 Feb 2022 18:13:16 +0000 (+0000) Subject: Merge remote-tracking branch 'upstream/pull/3398' X-Git-Tag: live~2651 X-Git-Url: https://git.openstreetmap.org/rails.git/commitdiff_plain/1f8df781be6788a43e0a76a9874366930473f64c?hp=6c1d73a509a1ac2216a704f3dc5534d573e5d7ae Merge remote-tracking branch 'upstream/pull/3398' --- diff --git a/CONFIGURE.md b/CONFIGURE.md index f7638a49a..9bf5e16bb 100644 --- a/CONFIGURE.md +++ b/CONFIGURE.md @@ -30,9 +30,7 @@ If you create a user by signing up to your local website, you need to confirm th $ bundle exec rails console >> user = User.find_by(:display_name => "My New User Name") => #[ ... ] ->> user.status = "active" -=> "active" ->> user.save! +>> user.activate! => true >> quit ``` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 474c80b4f..aaa1b5d4b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -38,8 +38,8 @@ bundle exec rails test:all You can view test coverage statistics by browsing the `coverage` directory. -The tests are automatically run on Pull Requests and other commits with the -results shown on [Travis CI](https://travis-ci.org/openstreetmap/openstreetmap-website). +The tests are automatically run on Pull Requests and other commits via github +actions. The results shown are within the PR display on github. ## Static Analysis @@ -79,14 +79,6 @@ database, and update the list of available keys manually. Adding or removing keys to this list is therefore discouraged, but contributions to the descriptive texts are welcome. -## Code Documentation - -To generate the HTML documentation of the API/rails code, run the command - -``` -rake doc:app -``` - ## Committing When you submit patches, the project maintainer has to read them and diff --git a/Gemfile b/Gemfile index 7bb5d7054..8782fdfef 100644 --- a/Gemfile +++ b/Gemfile @@ -1,7 +1,7 @@ source "https://rubygems.org" # Require rails -gem "rails", "6.1.4.4" +gem "rails", "7.0.2.2" # Require json for multi_json gem "json" @@ -43,7 +43,7 @@ gem "active_record_union" gem "bootstrap", "~> 4.5.0" gem "bootstrap_form", "~> 4.0" gem "cancancan" -gem "composite_primary_keys", "~> 13.0.0" +gem "composite_primary_keys", "~> 14.0.0" gem "config" gem "delayed_job_active_record" gem "http_accept_language", "~> 2.1.1" @@ -51,7 +51,7 @@ gem "i18n-js", ">= 3.0.0" gem "oauth-plugin", ">= 0.5.1" gem "openstreetmap-deadlock_retry", ">= 1.3.1", :require => "deadlock_retry" gem "rack-cors" -gem "rails-i18n", "~> 6.0.0" +gem "rails-i18n", "~> 7.0.0" gem "rinku", ">= 2.0.6", :require => "rails_rinku" gem "strong_migrations" gem "validates_email_format_of", ">= 1.5.1" @@ -151,7 +151,7 @@ group :test do gem "rubocop-performance" gem "rubocop-rails" gem "rubocop-rake" - gem "selenium-webdriver", "~> 3.142.7" + gem "selenium-webdriver" gem "simplecov", :require => false gem "simplecov-lcov", :require => false gem "webmock" diff --git a/Gemfile.lock b/Gemfile.lock index d86a7cf9f..0c8b03bda 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -3,75 +3,81 @@ GEM specs: aasm (5.2.0) concurrent-ruby (~> 1.0) - actioncable (6.1.4.4) - actionpack (= 6.1.4.4) - activesupport (= 6.1.4.4) + actioncable (7.0.2.2) + actionpack (= 7.0.2.2) + activesupport (= 7.0.2.2) nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailbox (6.1.4.4) - actionpack (= 6.1.4.4) - activejob (= 6.1.4.4) - activerecord (= 6.1.4.4) - activestorage (= 6.1.4.4) - activesupport (= 6.1.4.4) + actionmailbox (7.0.2.2) + actionpack (= 7.0.2.2) + activejob (= 7.0.2.2) + activerecord (= 7.0.2.2) + activestorage (= 7.0.2.2) + activesupport (= 7.0.2.2) mail (>= 2.7.1) - actionmailer (6.1.4.4) - actionpack (= 6.1.4.4) - actionview (= 6.1.4.4) - activejob (= 6.1.4.4) - activesupport (= 6.1.4.4) + net-imap + net-pop + net-smtp + actionmailer (7.0.2.2) + actionpack (= 7.0.2.2) + actionview (= 7.0.2.2) + activejob (= 7.0.2.2) + activesupport (= 7.0.2.2) mail (~> 2.5, >= 2.5.4) + net-imap + net-pop + net-smtp rails-dom-testing (~> 2.0) - actionpack (6.1.4.4) - actionview (= 6.1.4.4) - activesupport (= 6.1.4.4) - rack (~> 2.0, >= 2.0.9) + actionpack (7.0.2.2) + actionview (= 7.0.2.2) + activesupport (= 7.0.2.2) + rack (~> 2.0, >= 2.2.0) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.2.0) actionpack-page_caching (1.2.4) actionpack (>= 4.0.0) - actiontext (6.1.4.4) - actionpack (= 6.1.4.4) - activerecord (= 6.1.4.4) - activestorage (= 6.1.4.4) - activesupport (= 6.1.4.4) + actiontext (7.0.2.2) + actionpack (= 7.0.2.2) + activerecord (= 7.0.2.2) + activestorage (= 7.0.2.2) + activesupport (= 7.0.2.2) + globalid (>= 0.6.0) nokogiri (>= 1.8.5) - actionview (6.1.4.4) - activesupport (= 6.1.4.4) + actionview (7.0.2.2) + activesupport (= 7.0.2.2) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.1, >= 1.2.0) active_record_union (1.3.0) activerecord (>= 4.0) - activejob (6.1.4.4) - activesupport (= 6.1.4.4) + activejob (7.0.2.2) + activesupport (= 7.0.2.2) globalid (>= 0.3.6) - activemodel (6.1.4.4) - activesupport (= 6.1.4.4) - activerecord (6.1.4.4) - activemodel (= 6.1.4.4) - activesupport (= 6.1.4.4) + activemodel (7.0.2.2) + activesupport (= 7.0.2.2) + activerecord (7.0.2.2) + activemodel (= 7.0.2.2) + activesupport (= 7.0.2.2) activerecord-import (1.3.0) activerecord (>= 4.2) - activestorage (6.1.4.4) - actionpack (= 6.1.4.4) - activejob (= 6.1.4.4) - activerecord (= 6.1.4.4) - activesupport (= 6.1.4.4) - marcel (~> 1.0.0) + activestorage (7.0.2.2) + actionpack (= 7.0.2.2) + activejob (= 7.0.2.2) + activerecord (= 7.0.2.2) + activesupport (= 7.0.2.2) + marcel (~> 1.0) mini_mime (>= 1.1.0) - activesupport (6.1.4.4) + activesupport (7.0.2.2) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) tzinfo (~> 2.0) - zeitwerk (~> 2.3) addressable (2.8.0) public_suffix (>= 2.0.2, < 5.0) - annotate (3.1.1) - activerecord (>= 3.2, < 7.0) + annotate (3.2.0) + activerecord (>= 3.2, < 8.0) rake (>= 10.4, < 14.0) argon2 (2.1.1) ffi (~> 1.14) @@ -80,8 +86,8 @@ GEM autoprefixer-rails (10.4.2.0) execjs (~> 2) aws-eventstream (1.2.0) - aws-partitions (1.553.0) - aws-sdk-core (3.126.0) + aws-partitions (1.554.0) + aws-sdk-core (3.126.1) aws-eventstream (~> 1, >= 1.0.2) aws-partitions (~> 1, >= 1.525.0) aws-sigv4 (~> 1.1) @@ -124,8 +130,8 @@ GEM bzip2-ffi (1.1.0) ffi (~> 1.0) cancancan (3.3.0) - canonical-rails (0.2.13) - rails (>= 4.1, <= 7.0) + canonical-rails (0.2.14) + rails (>= 4.1, <= 7.1) capybara (3.36.0) addressable matrix @@ -135,19 +141,19 @@ GEM rack-test (>= 0.6.3) regexp_parser (>= 1.5, < 3.0) xpath (~> 3.2) - childprocess (3.0.0) + childprocess (4.1.0) coderay (1.1.3) - composite_primary_keys (13.0.3) - activerecord (~> 6.1.0) + composite_primary_keys (14.0.4) + activerecord (~> 7.0.2) concurrent-ruby (1.1.9) - config (3.1.1) + config (4.0.0) deep_merge (~> 1.2, >= 1.2.1) dry-validation (~> 1.0, >= 1.0.0) connection_pool (2.2.5) crack (0.4.5) rexml crass (1.0.6) - dalli (3.2.0) + dalli (3.2.1) debug_inspector (1.1.0) deep_merge (1.2.2) delayed_job (4.1.10) @@ -155,6 +161,7 @@ GEM delayed_job_active_record (4.1.7) activerecord (>= 3.0, < 8.0) delayed_job (>= 3.0, < 5) + digest (3.1.0) docile (1.4.0) doorkeeper (5.5.4) railties (>= 5) @@ -173,7 +180,7 @@ GEM dry-logic (1.2.0) concurrent-ruby (~> 1.0) dry-core (~> 0.5, >= 0.5) - dry-schema (1.8.0) + dry-schema (1.9.0) concurrent-ruby (~> 1.0) dry-configurable (~> 0.13, >= 0.13.0) dry-core (~> 0.5, >= 0.5) @@ -247,7 +254,7 @@ GEM html_tokenizer (0.0.7) htmlentities (4.3.4) http_accept_language (2.1.1) - i18n (1.9.1) + i18n (1.10.0) concurrent-ruby (~> 1.0) i18n-js (3.9.1) i18n (>= 0.6.6) @@ -266,10 +273,11 @@ GEM ruby-vips (>= 2.0.17, < 3) image_size (3.0.1) in_threads (1.6.0) + io-wait (0.2.1) jbuilder (2.11.5) actionview (>= 5.0.0) activesupport (>= 5.0.0) - jmespath (1.5.0) + jmespath (1.6.0) jquery-rails (4.4.0) rails-dom-testing (>= 1, < 3) railties (>= 4.2.0) @@ -286,7 +294,7 @@ GEM logstasher (2.1.5) activesupport (>= 5.2) request_store - loofah (2.13.0) + loofah (2.14.0) crass (~> 1.0.2) nokogiri (>= 1.5.9) mail (2.7.1) @@ -299,10 +307,25 @@ GEM mini_mime (1.1.2) mini_portile2 (2.7.1) minitest (5.15.0) - msgpack (1.4.4) + msgpack (1.4.5) multi_json (1.15.0) multi_xml (0.6.0) multipart-post (2.1.1) + net-imap (0.2.3) + digest + net-protocol + strscan + net-pop (0.1.1) + digest + net-protocol + timeout + net-protocol (0.1.2) + io-wait + timeout + net-smtp (0.3.1) + digest + net-protocol + timeout nio4r (2.5.8) nokogiri (1.13.1) mini_portile2 (~> 2.7.0) @@ -355,11 +378,11 @@ GEM parallel (1.21.0) parser (3.1.0.0) ast (~> 2.4.1) - pg (1.3.1) + pg (1.3.2) popper_js (1.16.0) progress (3.6.0) public_suffix (4.0.6) - puma (5.6.1) + puma (5.6.2) nio4r (~> 2.0) quad_tile (1.0.1) r2 (0.2.7) @@ -370,26 +393,25 @@ GEM rack-openid (1.4.2) rack (>= 1.1.0) ruby-openid (>= 2.1.8) - rack-protection (2.1.0) + rack-protection (2.2.0) rack rack-test (1.1.0) rack (>= 1.0, < 3) rack-uri_sanitizer (0.0.2) - rails (6.1.4.4) - actioncable (= 6.1.4.4) - actionmailbox (= 6.1.4.4) - actionmailer (= 6.1.4.4) - actionpack (= 6.1.4.4) - actiontext (= 6.1.4.4) - actionview (= 6.1.4.4) - activejob (= 6.1.4.4) - activemodel (= 6.1.4.4) - activerecord (= 6.1.4.4) - activestorage (= 6.1.4.4) - activesupport (= 6.1.4.4) + rails (7.0.2.2) + actioncable (= 7.0.2.2) + actionmailbox (= 7.0.2.2) + actionmailer (= 7.0.2.2) + actionpack (= 7.0.2.2) + actiontext (= 7.0.2.2) + actionview (= 7.0.2.2) + activejob (= 7.0.2.2) + activemodel (= 7.0.2.2) + activerecord (= 7.0.2.2) + activestorage (= 7.0.2.2) + activesupport (= 7.0.2.2) bundler (>= 1.15.0) - railties (= 6.1.4.4) - sprockets-rails (>= 2.0.0) + railties (= 7.0.2.2) rails-controller-testing (1.0.5) actionpack (>= 5.0.1.rc1) actionview (>= 5.0.1.rc1) @@ -399,21 +421,22 @@ GEM nokogiri (>= 1.6) rails-html-sanitizer (1.4.2) loofah (~> 2.3) - rails-i18n (6.0.0) + rails-i18n (7.0.2) i18n (>= 0.7, < 2) - railties (>= 6.0.0, < 7) - railties (6.1.4.4) - actionpack (= 6.1.4.4) - activesupport (= 6.1.4.4) + railties (>= 6.0.0, < 8) + railties (7.0.2.2) + actionpack (= 7.0.2.2) + activesupport (= 7.0.2.2) method_source - rake (>= 0.13) + rake (>= 12.2) thor (~> 1.0) + zeitwerk (~> 2.5) rainbow (3.1.1) rake (13.0.6) rb-fsevent (0.11.1) rb-inotify (0.10.1) ffi (~> 1.0) - regexp_parser (2.2.0) + regexp_parser (2.2.1) request_store (1.5.1) rack (>= 1.4) rexml (3.2.5) @@ -428,9 +451,9 @@ GEM rubocop-ast (>= 1.15.1, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 1.4.0, < 3.0) - rubocop-ast (1.15.1) + rubocop-ast (1.15.2) parser (>= 3.0.1.1) - rubocop-minitest (0.17.1) + rubocop-minitest (0.17.2) rubocop (>= 0.90, < 2.0) rubocop-performance (1.13.2) rubocop (>= 1.7.0, < 2.0) @@ -459,8 +482,9 @@ GEM sprockets-rails tilt secure_headers (6.3.3) - selenium-webdriver (3.142.7) - childprocess (>= 0.5, < 4.0) + selenium-webdriver (4.1.0) + childprocess (>= 0.5, < 5.0) + rexml (~> 3.2, >= 3.2.5) rubyzip (>= 1.2.2) simplecov (0.21.2) docile (~> 1.1) @@ -468,7 +492,7 @@ GEM simplecov_json_formatter (~> 0.1) simplecov-html (0.12.3) simplecov-lcov (0.8.0) - simplecov_json_formatter (0.1.3) + simplecov_json_formatter (0.1.4) smart_properties (1.17.0) sprockets (4.0.2) concurrent-ruby (~> 1.0) @@ -477,12 +501,14 @@ GEM actionpack (>= 5.2) activesupport (>= 5.2) sprockets (>= 3.0.0) - strong_migrations (0.7.9) - activerecord (>= 5) + strong_migrations (0.8.0) + activerecord (>= 5.2) + strscan (3.0.1) terser (1.1.8) execjs (>= 0.3.0, < 3) thor (1.2.1) tilt (2.0.10) + timeout (0.2.0) tzinfo (2.0.4) concurrent-ruby (~> 1.0) unicode-display_width (2.1.0) @@ -523,7 +549,7 @@ DEPENDENCIES cancancan canonical-rails capybara (>= 2.15) - composite_primary_keys (~> 13.0.0) + composite_primary_keys (~> 14.0.0) config connection_pool dalli @@ -568,9 +594,9 @@ DEPENDENCIES r2 (~> 0.2.7) rack-cors rack-uri_sanitizer - rails (= 6.1.4.4) + rails (= 7.0.2.2) rails-controller-testing - rails-i18n (~> 6.0.0) + rails-i18n (~> 7.0.0) rinku (>= 2.0.6) rotp rubocop @@ -581,7 +607,7 @@ DEPENDENCIES sanitize sassc-rails secure_headers - selenium-webdriver (~> 3.142.7) + selenium-webdriver simplecov simplecov-lcov strong_migrations diff --git a/INSTALL.md b/INSTALL.md index 1faf4bf22..68b0d120e 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -90,7 +90,7 @@ Installing other dependencies: * Install Homebrew from https://brew.sh/ * Install the latest version of Ruby: `brew install ruby` -* Install other dependencies: `brew install imagemagick libxml2 gd yarn pngcrush optipng pngquant jhead jpegoptim gifsicle svgo` +* Install other dependencies: `brew install imagemagick libxml2 gd yarn pngcrush optipng pngquant jhead jpegoptim gifsicle svgo advancecomp` * Install Bundler: `gem install bundler` (you might need to `sudo gem install bundler` if you get an error about permissions - or see note below about [developer Ruby setup](#rbenv)) You will need to tell `bundler` that `libxml2` is installed in a Homebrew location. If it uses the system-installed one then you will get errors installing the `libxml-ruby` gem later on. @@ -102,8 +102,7 @@ bundle config build.libxml-ruby --with-xml2-config=/usr/local/opt/libxml2/bin/xm If you want to run the tests, you need `geckodriver` as well: ``` -brew tap homebrew/cask -brew cask install geckodriver +brew install geckodriver ``` Note that OS X does not have a /home directory by default, so if you are using the GPX functions, you will need to change the directories specified in config/application.yml. diff --git a/app/controllers/api/traces_controller.rb b/app/controllers/api/traces_controller.rb index 9894441ff..aa8a06000 100644 --- a/app/controllers/api/traces_controller.rb +++ b/app/controllers/api/traces_controller.rb @@ -54,6 +54,8 @@ module Api send_data(trace.xml_file.read, :filename => "#{trace.id}.xml", :type => request.format.to_s, :disposition => "attachment") elsif request.format == Mime[:gpx] send_data(trace.xml_file.read, :filename => "#{trace.id}.gpx", :type => request.format.to_s, :disposition => "attachment") + elsif trace.file.attached? + redirect_to rails_blob_path(trace.file, :disposition => "attachment") else send_file(trace.trace_name, :filename => "#{trace.id}#{trace.extension_name}", :type => trace.mime_type, :disposition => "attachment") end @@ -97,12 +99,6 @@ module Api # Sanitise the user's filename name = file.original_filename.gsub(/[^a-zA-Z0-9.]/, "_") - # Get a temporary filename... - filename = "/tmp/#{rand}" - - # ...and save the uploaded file to that location - File.binwrite(filename, file.read) - # Create the trace object, falsely marked as already # inserted to stop the import daemon trying to load it trace = Trace.new( @@ -110,40 +106,14 @@ module Api :tagstring => tags, :description => description, :visibility => visibility, - :inserted => true, + :inserted => false, :user => current_user, - :timestamp => Time.now.getutc + :timestamp => Time.now.getutc, + :file => file ) - if trace.valid? - Trace.transaction do - begin - # Save the trace object - trace.save! - - # Rename the temporary file to the final name - FileUtils.mv(filename, trace.trace_name) - rescue StandardError - # Remove the file as we have failed to update the database - FileUtils.rm_f(filename) - - # Pass the exception on - raise - end - - begin - # Clear the inserted flag to make the import daemon load the trace - trace.inserted = false - trace.save! - rescue StandardError - # Remove the file as we have failed to update the database - FileUtils.rm_f(trace.trace_name) - - # Pass the exception on - raise - end - end - end + # Save the trace object + trace.save! # Finally save the user's preferred privacy level if pref = current_user.preferences.where(:k => "gps.trace.visibility").first diff --git a/app/controllers/traces_controller.rb b/app/controllers/traces_controller.rb index 43977c3ce..a9dbc8539 100644 --- a/app/controllers/traces_controller.rb +++ b/app/controllers/traces_controller.rb @@ -99,12 +99,8 @@ class TracesController < ApplicationController logger.info(params[:trace][:gpx_file].class.name) if params[:trace][:gpx_file].respond_to?(:read) - begin - @trace = do_create(params[:trace][:gpx_file], params[:trace][:tagstring], - params[:trace][:description], params[:trace][:visibility]) - rescue StandardError => e - logger.debug e - end + @trace = do_create(params[:trace][:gpx_file], params[:trace][:tagstring], + params[:trace][:description], params[:trace][:visibility]) if @trace.id flash[:notice] = t ".trace_uploaded" @@ -141,6 +137,8 @@ class TracesController < ApplicationController send_data(trace.xml_file.read, :filename => "#{trace.id}.xml", :type => request.format.to_s, :disposition => "attachment") elsif request.format == Mime[:gpx] send_data(trace.xml_file.read, :filename => "#{trace.id}.gpx", :type => request.format.to_s, :disposition => "attachment") + elsif trace.file.attached? + redirect_to rails_blob_path(trace.file, :disposition => "attachment") else send_file(trace.trace_name, :filename => "#{trace.id}#{trace.extension_name}", :type => trace.mime_type, :disposition => "attachment") end @@ -217,8 +215,12 @@ class TracesController < ApplicationController if trace.visible? && trace.inserted? if trace.public? || (current_user && current_user == trace.user) - expires_in 7.days, :private => !trace.public?, :public => trace.public? - send_file(trace.large_picture_name, :filename => "#{trace.id}.gif", :type => "image/gif", :disposition => "inline") + if trace.icon.attached? + redirect_to rails_blob_path(trace.image, :disposition => "inline") + else + expires_in 7.days, :private => !trace.public?, :public => trace.public? + send_file(trace.large_picture_name, :filename => "#{trace.id}.gif", :type => "image/gif", :disposition => "inline") + end else head :forbidden end @@ -234,8 +236,12 @@ class TracesController < ApplicationController if trace.visible? && trace.inserted? if trace.public? || (current_user && current_user == trace.user) - expires_in 7.days, :private => !trace.public?, :public => trace.public? - send_file(trace.icon_picture_name, :filename => "#{trace.id}_icon.gif", :type => "image/gif", :disposition => "inline") + if trace.icon.attached? + redirect_to rails_blob_path(trace.icon, :disposition => "inline") + else + expires_in 7.days, :private => !trace.public?, :public => trace.public? + send_file(trace.icon_picture_name, :filename => "#{trace.id}_icon.gif", :type => "image/gif", :disposition => "inline") + end else head :forbidden end @@ -252,62 +258,29 @@ class TracesController < ApplicationController # Sanitise the user's filename name = file.original_filename.gsub(/[^a-zA-Z0-9.]/, "_") - # Get a temporary filename... - filename = "/tmp/#{rand}" - - # ...and save the uploaded file to that location - File.binwrite(filename, file.read) - - # Create the trace object, falsely marked as already - # inserted to stop the import daemon trying to load it + # Create the trace object trace = Trace.new( :name => name, :tagstring => tags, :description => description, :visibility => visibility, - :inserted => true, + :inserted => false, :user => current_user, - :timestamp => Time.now.getutc + :timestamp => Time.now.getutc, + :file => file ) - if trace.valid? - Trace.transaction do - begin - # Save the trace object - trace.save! - - # Rename the temporary file to the final name - FileUtils.mv(filename, trace.trace_name) - rescue StandardError - # Remove the file as we have failed to update the database - FileUtils.rm_f(filename) - - # Pass the exception on - raise - end - - begin - # Clear the inserted flag to make the import daemon load the trace - trace.inserted = false - trace.save! - rescue StandardError - # Remove the file as we have failed to update the database - FileUtils.rm_f(trace.trace_name) - - # Pass the exception on - raise - end + # Save the trace object + if trace.save + # Finally save the user's preferred privacy level + if pref = current_user.preferences.where(:k => "gps.trace.visibility").first + pref.v = visibility + pref.save + else + current_user.preferences.create(:k => "gps.trace.visibility", :v => visibility) end end - # Finally save the user's preferred privacy level - if pref = current_user.preferences.where(:k => "gps.trace.visibility").first - pref.v = visibility - pref.save - else - current_user.preferences.create(:k => "gps.trace.visibility", :v => visibility) - end - trace end diff --git a/app/models/relation.rb b/app/models/relation.rb index a231feddb..4200a08dd 100644 --- a/app/models/relation.rb +++ b/app/models/relation.rb @@ -206,6 +206,8 @@ class Relation < ApplicationRecord end def preconditions_ok?(good_members = []) + raise OSM::APITooManyRelationMembersError.new(id, members.length, Settings.max_number_of_relation_members) if members.length > Settings.max_number_of_relation_members + # These are hastables that store an id in the index of all # the nodes/way/relations that have already been added. # If the member is valid and visible then we add it to the diff --git a/app/models/trace.rb b/app/models/trace.rb index b3d87fc09..bdafdd9d5 100644 --- a/app/models/trace.rb +++ b/app/models/trace.rb @@ -39,6 +39,10 @@ class Trace < ApplicationRecord scope :visible_to_all, -> { where(:visibility => %w[public identifiable]) } scope :tagged, ->(t) { joins(:tags).where(:gpx_file_tags => { :tag => t }) } + has_one_attached :file, :service => Settings.trace_file_storage + has_one_attached :image, :service => Settings.trace_image_storage + has_one_attached :icon, :service => Settings.trace_icon_storage + validates :user, :presence => true, :associated => true validates :name, :presence => true, :length => 1..255, :characters => true validates :description, :presence => { :on => :create }, :length => 1..255, :characters => true @@ -46,6 +50,7 @@ class Trace < ApplicationRecord validates :visibility, :inclusion => %w[private public trackable identifiable] after_destroy :remove_files + after_save :set_filename def tagstring tags.collect(&:tag).join(", ") @@ -68,6 +73,18 @@ class Trace < ApplicationRecord end end + def file=(attachable) + case attachable + when ActionDispatch::Http::UploadedFile, Rack::Test::UploadedFile + super(:io => attachable, + :filename => attachable.original_filename, + :content_type => content_type(attachable.path), + :identify => false) + else + super(attachable) + end + end + def public? visibility == "public" || visibility == "identifiable" end @@ -80,29 +97,27 @@ class Trace < ApplicationRecord visibility == "identifiable" end - def large_picture=(data) - f = File.new(large_picture_name, "wb") - f.syswrite(data) - f.close - end - - def icon_picture=(data) - f = File.new(icon_picture_name, "wb") - f.syswrite(data) - f.close - end - def large_picture - f = File.new(large_picture_name, "rb") - data = f.sysread(File.size(f.path)) - f.close + if image.attached? + data = image.blob.download + else + f = File.new(large_picture_name, "rb") + data = f.sysread(File.size(f.path)) + f.close + end + data end def icon_picture - f = File.new(icon_picture_name, "rb") - data = f.sysread(File.size(f.path)) - f.close + if icon.attached? + data = icon.blob.download + else + f = File.new(icon_picture_name, "rb") + data = f.sysread(File.size(f.path)) + f.close + end + data end @@ -119,46 +134,22 @@ class Trace < ApplicationRecord end def mime_type - filetype = Open3.capture2("/usr/bin/file", "-Lbz", trace_name).first.chomp - gzipped = filetype.include?("gzip compressed") - bzipped = filetype.include?("bzip2 compressed") - zipped = filetype.include?("Zip archive") - tarred = filetype.include?("tar archive") - - if gzipped - "application/x-gzip" - elsif bzipped - "application/x-bzip2" - elsif zipped - "application/x-zip" - elsif tarred - "application/x-tar" + if file.attached? + file.content_type else - "application/gpx+xml" + content_type(trace_name) end end def extension_name - filetype = Open3.capture2("/usr/bin/file", "-Lbz", trace_name).first.chomp - gzipped = filetype.include?("gzip compressed") - bzipped = filetype.include?("bzip2 compressed") - zipped = filetype.include?("Zip archive") - tarred = filetype.include?("tar archive") - - if tarred && gzipped - ".tar.gz" - elsif tarred && bzipped - ".tar.bz2" - elsif tarred - ".tar" - elsif gzipped - ".gpx.gz" - elsif bzipped - ".gpx.bz2" - elsif zipped - ".zip" - else - ".gpx" + case mime_type + when "application/x-tar+gzip" then ".tar.gz" + when "application/x-tar+x-bzip2" then ".tar.bz2" + when "application/x-tar" then ".tar" + when "application/zip" then ".zip" + when "application/gzip" then ".gpx.gz" + when "application/x-bzip2" then ".gpx.bz2" + else ".gpx" end end @@ -207,106 +198,156 @@ class Trace < ApplicationRecord end def xml_file - filetype = Open3.capture2("/usr/bin/file", "-Lbz", trace_name).first.chomp - gzipped = filetype.include?("gzip compressed") - bzipped = filetype.include?("bzip2 compressed") - zipped = filetype.include?("Zip archive") - tarred = filetype.include?("tar archive") - - if gzipped || bzipped || zipped || tarred - file = Tempfile.new("trace.#{id}") - - if tarred && gzipped - system("tar", "-zxOf", trace_name, :out => file.path) - elsif tarred && bzipped - system("tar", "-jxOf", trace_name, :out => file.path) - elsif tarred - system("tar", "-xOf", trace_name, :out => file.path) - elsif gzipped - system("gunzip", "-c", trace_name, :out => file.path) - elsif bzipped - system("bunzip2", "-c", trace_name, :out => file.path) - elsif zipped - system("unzip", "-p", trace_name, "-x", "__MACOSX/*", :out => file.path, :err => "/dev/null") + with_trace_file do |trace_name| + filetype = Open3.capture2("/usr/bin/file", "-Lbz", trace_name).first.chomp + gzipped = filetype.include?("gzip compressed") + bzipped = filetype.include?("bzip2 compressed") + zipped = filetype.include?("Zip archive") + tarred = filetype.include?("tar archive") + + if gzipped || bzipped || zipped || tarred + file = Tempfile.new("trace.#{id}") + + if tarred && gzipped + system("tar", "-zxOf", trace_name, :out => file.path) + elsif tarred && bzipped + system("tar", "-jxOf", trace_name, :out => file.path) + elsif tarred + system("tar", "-xOf", trace_name, :out => file.path) + elsif gzipped + system("gunzip", "-c", trace_name, :out => file.path) + elsif bzipped + system("bunzip2", "-c", trace_name, :out => file.path) + elsif zipped + system("unzip", "-p", trace_name, "-x", "__MACOSX/*", :out => file.path, :err => "/dev/null") + end + + file.unlink + else + file = File.open(trace_name) end - file.unlink - else - file = File.open(trace_name) + file end - - file end def import logger.info("GPX Import importing #{name} (#{id}) from #{user.email}") - gpx = GPX::File.new(trace_name) - - f_lat = 0 - f_lon = 0 - first = true - - # If there are any existing points for this trace then delete them - Tracepoint.where(:gpx_id => id).delete_all - - gpx.points.each_slice(1_000) do |points| - # Gather the trace points together for a bulk import - tracepoints = [] + with_trace_file do |trace_name| + gpx = GPX::File.new(trace_name) + + f_lat = 0 + f_lon = 0 + first = true + + # If there are any existing points for this trace then delete them + Tracepoint.where(:gpx_id => id).delete_all + + gpx.points.each_slice(1_000) do |points| + # Gather the trace points together for a bulk import + tracepoints = [] + + points.each do |point| + if first + f_lat = point.latitude + f_lon = point.longitude + first = false + end + + tp = Tracepoint.new + tp.lat = point.latitude + tp.lon = point.longitude + tp.altitude = point.altitude + tp.timestamp = point.timestamp + tp.gpx_id = id + tp.trackid = point.segment + tracepoints << tp + end - points.each do |point| - if first - f_lat = point.latitude - f_lon = point.longitude - first = false + # Run the before_save and before_create callbacks, and then import them in bulk with activerecord-import + tracepoints.each do |tp| + tp.run_callbacks(:save) { false } + tp.run_callbacks(:create) { false } end - tp = Tracepoint.new - tp.lat = point.latitude - tp.lon = point.longitude - tp.altitude = point.altitude - tp.timestamp = point.timestamp - tp.gpx_id = id - tp.trackid = point.segment - tracepoints << tp + Tracepoint.import!(tracepoints) end - # Run the before_save and before_create callbacks, and then import them in bulk with activerecord-import - tracepoints.each do |tp| - tp.run_callbacks(:save) { false } - tp.run_callbacks(:create) { false } + if gpx.actual_points.positive? + max_lat = Tracepoint.where(:gpx_id => id).maximum(:latitude) + min_lat = Tracepoint.where(:gpx_id => id).minimum(:latitude) + max_lon = Tracepoint.where(:gpx_id => id).maximum(:longitude) + min_lon = Tracepoint.where(:gpx_id => id).minimum(:longitude) + + max_lat = max_lat.to_f / 10000000 + min_lat = min_lat.to_f / 10000000 + max_lon = max_lon.to_f / 10000000 + min_lon = min_lon.to_f / 10000000 + + self.latitude = f_lat + self.longitude = f_lon + image.attach(:io => gpx.picture(min_lat, min_lon, max_lat, max_lon, gpx.actual_points), :filename => "#{id}.gif", :content_type => "image/gif") + icon.attach(:io => gpx.icon(min_lat, min_lon, max_lat, max_lon), :filename => "#{id}_icon.gif", :content_type => "image/gif") + self.size = gpx.actual_points + self.inserted = true + save! end - Tracepoint.import!(tracepoints) + logger.info "done trace #{id}" + + gpx end + end - if gpx.actual_points.positive? - max_lat = Tracepoint.where(:gpx_id => id).maximum(:latitude) - min_lat = Tracepoint.where(:gpx_id => id).minimum(:latitude) - max_lon = Tracepoint.where(:gpx_id => id).maximum(:longitude) - min_lon = Tracepoint.where(:gpx_id => id).minimum(:longitude) - - max_lat = max_lat.to_f / 10000000 - min_lat = min_lat.to_f / 10000000 - max_lon = max_lon.to_f / 10000000 - min_lon = min_lon.to_f / 10000000 - - self.latitude = f_lat - self.longitude = f_lon - self.large_picture = gpx.picture(min_lat, min_lon, max_lat, max_lon, gpx.actual_points) - self.icon_picture = gpx.icon(min_lat, min_lon, max_lat, max_lon) - self.size = gpx.actual_points - self.inserted = true - save! + def migrate_to_storage! + file.attach(:io => File.open(trace_name), + :filename => name, + :content_type => content_type(trace_name), + :identify => false) + + if inserted + image.attach(:io => File.open(large_picture_name), + :filename => "#{id}.gif", + :content_type => "image/gif") + icon.attach(:io => File.open(icon_picture_name), + :filename => "#{id}_icon.gif", + :content_type => "image/gif") end - logger.info "done trace #{id}" + save! - gpx + remove_files end private + def content_type(file) + case Open3.capture2("/usr/bin/file", "-Lbz", file).first.chomp + when /.*\btar archive\b.*\bgzip\b/ then "application/x-tar+gzip" + when /.*\btar archive\b.*\bbzip2\b/ then "application/x-tar+x-bzip2" + when /.*\btar archive\b/ then "application/x-tar" + when /.*\bZip archive\b/ then "application/zip" + when /.*\bXML\b.*\bgzip\b/ then "application/gzip" + when /.*\bXML\b.*\bbzip2\b/ then "application/x-bzip2" + else "application/gpx+xml" + end + end + + def with_trace_file + if file.attached? + file.open do |file| + yield file.path + end + else + yield trace_name + end + end + + def set_filename + file.blob.update(:filename => "#{id}#{extension_name}") if file.attached? + end + def remove_files FileUtils.rm_f(trace_name) FileUtils.rm_f(icon_picture_name) diff --git a/app/models/user.rb b/app/models/user.rb index 722d65302..7a0d06992 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -87,7 +87,7 @@ class User < ApplicationRecord scope :active, -> { where(:status => %w[active confirmed]) } scope :identifiable, -> { where(:data_public => true) } - has_one_attached :avatar + has_one_attached :avatar, :service => Settings.avatar_storage validates :display_name, :presence => true, :length => 3..255, :exclusion => %w[new terms save confirm confirm-email go_public reset-password forgot-password suspended] diff --git a/app/views/api/capabilities/show.builder b/app/views/api/capabilities/show.builder index 682373898..b6a38723d 100644 --- a/app/views/api/capabilities/show.builder +++ b/app/views/api/capabilities/show.builder @@ -6,6 +6,7 @@ xml.osm(OSM::API.new.xml_root_attributes) do |osm| api.note_area(:maximum => Settings.max_note_request_area) api.tracepoints(:per_page => Settings.tracepoints_per_page) api.waynodes(:maximum => Settings.max_number_of_way_nodes) + api.relationmembers(:maximum => Settings.max_number_of_relation_members) api.changesets(:maximum_elements => Changeset::MAX_ELEMENTS) api.timeout(:seconds => Settings.api_timeout) api.status(:database => @database_status, diff --git a/bin/setup b/bin/setup index d0b0fbd57..ec47b79b3 100755 --- a/bin/setup +++ b/bin/setup @@ -17,12 +17,9 @@ FileUtils.chdir APP_ROOT do system! "gem install bundler --conservative" system("bundle check") || system!("bundle install") - # Install JavaScript dependencies - system! "bin/yarn" - # puts "\n== Copying sample files ==" - # unless File.exist?('config/database.yml') - # FileUtils.cp 'config/database.yml.sample', 'config/database.yml' + # unless File.exist?("config/database.yml") + # FileUtils.cp "config/database.yml.sample", "config/database.yml" # end puts "\n== Preparing database ==" diff --git a/config/environments/development.rb b/config/environments/development.rb index ab4f00220..750a596b4 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -14,6 +14,9 @@ Rails.application.configure do # Show full error reports. config.consider_all_requests_local = true + # Enable server timing + config.server_timing = true + # Enable/disable caching. By default caching is disabled. # Run rails dev:cache to toggle caching. if Rails.root.join("tmp/caching-dev.txt").exist? @@ -53,11 +56,6 @@ Rails.application.configure do # Highlight code that triggered database queries in logs. config.active_record.verbose_query_logs = true unless Settings.status == "database_offline" - # Debug mode disables concatenation and preprocessing of assets. - # This option may cause significant delays in view rendering with a large - # number of complex assets. - config.assets.debug = true - # Suppress logger output for asset requests. config.assets.quiet = true @@ -70,10 +68,6 @@ Rails.application.configure do # Annotate rendered view with file names. # config.action_view.annotate_rendered_view_with_filenames = true - # Use an evented file watcher to asynchronously detect changes in source code, - # routes, locales, etc. This feature depends on the listen gem. - config.file_watcher = ActiveSupport::EventedFileUpdateChecker - # Uncomment if you wish to allow Action Cable access from any origin. # config.action_cable.disable_request_forgery_protection = true diff --git a/config/environments/production.rb b/config/environments/production.rb index 5c72d449c..7dd3922a1 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -32,19 +32,19 @@ Rails.application.configure do config.assets.compile = false # Enable serving of images, stylesheets, and JavaScripts from an asset server. - # config.asset_host = 'http://assets.example.com' + # config.asset_host = "http://assets.example.com" # Specifies the header that your server uses for sending files. - # config.action_dispatch.x_sendfile_header = 'X-Sendfile' # for Apache - # config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for NGINX + # config.action_dispatch.x_sendfile_header = "X-Sendfile" # for Apache + # config.action_dispatch.x_sendfile_header = "X-Accel-Redirect" # for NGINX # Store uploaded files on the local file system (see config/storage.yml for options). - config.active_storage.service = Settings.storage_service.to_sym + config.active_storage.service = :local # Mount Action Cable outside main process or domain. # config.action_cable.mount_path = nil - # config.action_cable.url = 'wss://example.com/cable' - # config.action_cable.allowed_request_origins = [ 'http://example.com', /http:\/\/example.*/ ] + # config.action_cable.url = "wss://example.com/cable" + # config.action_cable.allowed_request_origins = [ "http://example.com", /http:\/\/example.*/ ] # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. # config.force_ssl = true @@ -79,21 +79,15 @@ Rails.application.configure do # the I18n.default_locale when a translation cannot be found). config.i18n.fallbacks = true - # Send deprecation notices to registered listeners. - config.active_support.deprecation = :notify - - # Log disallowed deprecations. - config.active_support.disallowed_deprecation = :log - - # Tell Active Support which deprecation messages to disallow. - config.active_support.disallowed_deprecation_warnings = [] + # Don't log any deprecations. + config.active_support.report_deprecations = false # Use default logging formatter so that PID and timestamp are not suppressed. config.log_formatter = ::Logger::Formatter.new # Use a different logger for distributed setups. - # require 'syslog/logger' - # config.logger = ActiveSupport::TaggedLogging.new(Syslog::Logger.new 'app-name') + # require "syslog/logger" + # config.logger = ActiveSupport::TaggedLogging.new(Syslog::Logger.new "app-name") if ENV["RAILS_LOG_TO_STDOUT"].present? logger = ActiveSupport::Logger.new($stdout) @@ -104,27 +98,6 @@ Rails.application.configure do # Do not dump schema after migrations. config.active_record.dump_schema_after_migration = false unless Settings.status == "database_offline" - # Inserts middleware to perform automatic connection switching. - # The `database_selector` hash is used to pass options to the DatabaseSelector - # middleware. The `delay` is used to determine how long to wait after a write - # to send a subsequent read to the primary. - # - # The `database_resolver` class is used by the middleware to determine which - # database is appropriate to use based on the time delay. - # - # The `database_resolver_context` class is used by the middleware to set - # timestamps for the last write to the primary. The resolver uses the context - # class timestamps to determine how long to wait before reading from the - # replica. - # - # By default Rails will store a last write timestamp in the session. The - # DatabaseSelector middleware is designed as such you can define your own - # strategy for connection switching and pass that into the middleware through - # these configuration options. - # config.active_record.database_selector = { delay: 2.seconds } - # config.active_record.database_resolver = ActiveRecord::Middleware::DatabaseSelector::Resolver - # config.active_record.database_resolver_context = ActiveRecord::Middleware::DatabaseSelector::Resolver::Session - # Enable autoloading of dependencies. config.enable_dependency_loading = true diff --git a/config/environments/test.rb b/config/environments/test.rb index 007f2a3b4..875716e50 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -8,12 +8,13 @@ require "active_support/core_ext/integer/time" Rails.application.configure do # Settings specified here will take precedence over those in config/application.rb. + # Turn false under Spring and add config.action_view.cache_template_loading = true config.cache_classes = true - # Do not eager load code on boot. This avoids loading your whole application - # just for the purpose of running a single test. If you are using a tool that - # preloads Rails for running tests, you may have to set it to true. - config.eager_load = false + # Eager loading loads your whole application. When running a single test locally, + # this probably isn't necessary. It's a good idea to do in a continuous integration + # system, or in some way before deploying your code. + config.eager_load = ENV["CI"].present? # Configure public file server for tests with Cache-Control for performance. config.public_file_server.enabled = true diff --git a/config/initializers/config.rb b/config/initializers/config.rb index d0f8c26fc..e51281e11 100644 --- a/config/initializers/config.rb +++ b/config/initializers/config.rb @@ -74,9 +74,13 @@ Config.setup do |config| required(:max_note_request_area).filled(:number?) required(:tracepoints_per_page).filled(:int?) required(:max_number_of_way_nodes).filled(:int?) + required(:max_number_of_relation_members).filled(:int?) required(:api_timeout).filled(:int?) required(:imagery_blacklist).maybe(:array?) required(:status).filled(:str?, :included_in? => ALLOWED_STATUS) - required(:storage_service).filled(:str?) + required(:avatar_storage).filled(:str?) + required(:trace_file_storage).filled(:str?) + required(:trace_image_storage).filled(:str?) + required(:trace_icon_storage).filled(:str?) end end diff --git a/config/initializers/content_security_policy.rb b/config/initializers/content_security_policy.rb index 35d0f26fc..3621f97f8 100644 --- a/config/initializers/content_security_policy.rb +++ b/config/initializers/content_security_policy.rb @@ -4,27 +4,23 @@ # For further information see the following documentation # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy -# Rails.application.config.content_security_policy do |policy| -# policy.default_src :self, :https -# policy.font_src :self, :https, :data -# policy.img_src :self, :https, :data -# policy.object_src :none -# policy.script_src :self, :https -# policy.style_src :self, :https -# # If you are using webpack-dev-server then specify webpack-dev-server host -# policy.connect_src :self, :https, "http://localhost:3035", "ws://localhost:3035" if Rails.env.development? - -# # Specify URI for violation reports -# # policy.report_uri "/csp-violation-report-endpoint" +# Rails.application.configure do +# config.content_security_policy do |policy| +# policy.default_src :self, :https +# policy.font_src :self, :https, :data +# policy.img_src :self, :https, :data +# policy.object_src :none +# policy.script_src :self, :https +# policy.style_src :self, :https +# # Specify URI for violation reports +# # policy.report_uri "/csp-violation-report-endpoint" +# end +# +# # Generate session nonces for permitted importmap and inline scripts +# config.content_security_policy_nonce_generator = ->(request) { request.session.id.to_s } +# config.content_security_policy_nonce_directives = %w(script-src) +# +# # Report CSP violations to a specified URI. See: +# # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy-Report-Only +# # config.content_security_policy_report_only = true # end - -# If you are using UJS then enable automatic nonce generation -# Rails.application.config.content_security_policy_nonce_generator = -> request { SecureRandom.base64(16) } - -# Set the nonce only to specific directives -# Rails.application.config.content_security_policy_nonce_directives = %w(script-src) - -# Report CSP violations to a specified URI -# For further information see the following documentation: -# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy-Report-Only -# Rails.application.config.content_security_policy_report_only = true diff --git a/config/initializers/inflections.rb b/config/initializers/inflections.rb index ac033bf9d..3860f659e 100644 --- a/config/initializers/inflections.rb +++ b/config/initializers/inflections.rb @@ -4,13 +4,13 @@ # are locale specific, and you may define rules for as many different # locales as you wish. All of these examples are active by default: # ActiveSupport::Inflector.inflections(:en) do |inflect| -# inflect.plural /^(ox)$/i, '\1en' -# inflect.singular /^(ox)en/i, '\1' -# inflect.irregular 'person', 'people' +# inflect.plural /^(ox)$/i, "\\1en" +# inflect.singular /^(ox)en/i, "\\1" +# inflect.irregular "person", "people" # inflect.uncountable %w( fish sheep ) # end # These inflection rules are supported but not enabled by default: # ActiveSupport::Inflector.inflections(:en) do |inflect| -# inflect.acronym 'RESTful' +# inflect.acronym "RESTful" # end diff --git a/config/initializers/new_framework_defaults_7_0.rb b/config/initializers/new_framework_defaults_7_0.rb new file mode 100644 index 000000000..a579326e2 --- /dev/null +++ b/config/initializers/new_framework_defaults_7_0.rb @@ -0,0 +1,117 @@ +# Be sure to restart your server when you modify this file. +# +# This file eases your Rails 7.0 framework defaults upgrade. +# +# Uncomment each configuration one by one to switch to the new default. +# Once your application is ready to run with all new defaults, you can remove +# this file and set the `config.load_defaults` to `7.0`. +# +# Read the Guide for Upgrading Ruby on Rails for more info on each option. +# https://guides.rubyonrails.org/upgrading_ruby_on_rails.html + +# `button_to` view helper will render `