From 9f8033214dfb6c156fb6238b39fef60f28783a66 Mon Sep 17 00:00:00 2001 From: Jeffrey Warren Date: Tue, 24 May 2022 18:57:17 -0400 Subject: [PATCH] =?UTF-8?q?migrate=20sentry-raven=20to=20sentry-ruby,=20up?= =?UTF-8?q?grade=20to=20Ruby=202.7.6,=20remove=20GDAL=20=F0=9F=98=B2=20(#1?= =?UTF-8?q?712)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * update application.rb * try to complete migration * fixed gemfile.lock * updated to Ruby 2.7.3 * more ruby 2.7.3 updates for dockerfiles * dockerfiles to 2.7.6 * remove local map exporting and gdal * more gdal removal * libappindicator3-7 \ * fixed libindicator3-7 * update for Debian 10 - libayatana-appindicator * all ruby 2.7.6 now * libayatana-appindicator in dockerfiles * remove system test deps for dev dockerfile --- .github/workflows/tests.yml | 16 +- Gemfile | 9 +- Gemfile.lock | 46 +-- config/application.rb | 18 +- dockerfiles/development | 31 +- dockerfiles/production | 7 +- install_cloud.sh | 28 -- lib/exporter-deps.sh | 9 +- lib/exporter.rb | 367 --------------------- test/controllers/export_controller_test.rb | 77 ----- test/models/export_test.rb | 21 -- test/models/exporter_test.rb | 57 ---- test/models/map_test.rb | 37 --- test/models/warpable_test.rb | 29 -- 14 files changed, 62 insertions(+), 690 deletions(-) delete mode 100755 install_cloud.sh delete mode 100644 lib/exporter.rb delete mode 100644 test/models/export_test.rb delete mode 100644 test/models/exporter_test.rb diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 97cbeef1..0d615704 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,7 +13,7 @@ jobs: - name: Prepare Ruby uses: ruby/setup-ruby@v1 with: - ruby-version: 2.4.6 + ruby-version: 2.7.6 bundler-cache: true - name: Rubocop report env: @@ -26,7 +26,7 @@ jobs: - uses: actions/checkout@v2 - uses: ruby/setup-ruby@v1 with: - ruby-version: 2.4.6 + ruby-version: 2.7.6 bundler-cache: true - uses: ./.github/actions/setup-test-environment - uses: egordm/gha-yarn-node-cache@v1 @@ -47,7 +47,7 @@ jobs: - uses: actions/checkout@v2 - uses: ruby/setup-ruby@v1 with: - ruby-version: 2.4.6 + ruby-version: 2.7.6 bundler-cache: true - uses: ./.github/actions/setup-test-environment - uses: egordm/gha-yarn-node-cache@v1 @@ -67,7 +67,7 @@ jobs: - uses: actions/checkout@v2 - uses: ruby/setup-ruby@v1 with: - ruby-version: 2.4.6 + ruby-version: 2.7.6 bundler-cache: true - uses: ./.github/actions/setup-test-environment - uses: egordm/gha-yarn-node-cache@v1 @@ -99,7 +99,7 @@ jobs: - uses: actions/checkout@v2 - uses: ruby/setup-ruby@v1 with: - ruby-version: 2.4.6 + ruby-version: 2.7.6 bundler-cache: true - uses: nanasess/setup-chromedriver@v1.0.7 - uses: ./.github/actions/setup-test-environment @@ -131,7 +131,7 @@ jobs: - uses: actions/checkout@v2 - uses: ruby/setup-ruby@v1 with: - ruby-version: 2.4.6 + ruby-version: 2.7.6 bundler-cache: true - uses: ./.github/actions/setup-test-environment - name: 'Development Docker Build' @@ -143,7 +143,7 @@ jobs: - uses: actions/checkout@v2 - uses: ruby/setup-ruby@v1 with: - ruby-version: 2.4.6 + ruby-version: 2.7.6 bundler-cache: true - uses: ./.github/actions/setup-test-environment - name: 'Development Docker Build' @@ -155,7 +155,7 @@ jobs: - uses: actions/checkout@v2 - uses: ruby/setup-ruby@v1 with: - ruby-version: 2.4.6 + ruby-version: 2.7.6 bundler-cache: true - uses: ./.github/actions/setup-test-environment - uses: egordm/gha-yarn-node-cache@v1 diff --git a/Gemfile b/Gemfile index 52ea6e70..92e6c212 100644 --- a/Gemfile +++ b/Gemfile @@ -1,11 +1,12 @@ source 'https://rubygems.org' -ruby '2.4.6' +ruby '2.7.6' gem 'rails', '~> 5.2.8' gem 'rake', '~> 13.0.1' gem 'tzinfo-data' -gem 'skylight' -gem 'sentry-raven' +gem 'skylight', '~> 5.0.0' +gem 'sentry-ruby' +gem 'sentry-rails' gem 'will_paginate', '3.3.1' gem 'will_paginate-bootstrap4', '~> 0.2.2' @@ -97,4 +98,4 @@ group :passenger do gem 'passenger' end -gem 'httparty' +gem 'httparty' \ No newline at end of file diff --git a/Gemfile.lock b/Gemfile.lock index 46fb4171..491daeb6 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -98,25 +98,6 @@ GEM execjs (2.8.1) faker (2.12.0) i18n (>= 1.6, < 2) - faraday (1.7.1) - faraday-em_http (~> 1.0) - faraday-em_synchrony (~> 1.0) - faraday-excon (~> 1.1) - faraday-httpclient (~> 1.0.1) - faraday-net_http (~> 1.0) - faraday-net_http_persistent (~> 1.1) - faraday-patron (~> 1.0) - faraday-rack (~> 1.0) - multipart-post (>= 1.2, < 3) - ruby2_keywords (>= 0.0.4) - faraday-em_http (1.0.0) - faraday-em_synchrony (1.0.0) - faraday-excon (1.1.0) - faraday-httpclient (1.0.1) - faraday-net_http (1.0.1) - faraday-net_http_persistent (1.2.0) - faraday-patron (1.0.0) - faraday-rack (1.0.0) ffi (1.15.5) friendly_id (5.4.2) activerecord (>= 4.0.0) @@ -174,7 +155,6 @@ GEM msgpack (1.4.5) multi_json (1.15.0) multi_xml (0.6.0) - multipart-post (2.1.1) mysql2 (0.5.4) net-http-persistent (2.9.4) nio4r (2.5.8) @@ -296,7 +276,6 @@ GEM ruby-openid (>= 2.1.7) ruby-prof (1.4.2) ruby-progressbar (1.11.0) - ruby2_keywords (0.0.5) rubyzip (2.3.2) sass (3.7.4) sass-listen (~> 4.0.0) @@ -314,16 +293,20 @@ GEM selenium-webdriver (3.142.7) childprocess (>= 0.5, < 4.0) rubyzip (>= 1.2.2) - sentry-raven (3.1.2) - faraday (>= 1.0) + sentry-rails (5.3.1) + railties (>= 5.0) + sentry-ruby-core (~> 5.3.1) + sentry-ruby (5.3.1) + concurrent-ruby (~> 1.0, >= 1.0.2) + sentry-ruby-core (= 5.3.1) + sentry-ruby-core (5.3.1) + concurrent-ruby simplecov (0.18.5) docile (~> 1.1) simplecov-html (~> 0.11) simplecov-html (0.12.3) - skylight (4.3.2) - skylight-core (= 4.3.2) - skylight-core (4.3.2) - activesupport (>= 4.2.0) + skylight (5.0.1) + activesupport (>= 5.2.0) spring (2.1.1) spring-watcher-listen (2.0.1) listen (>= 2.7, < 4.0) @@ -416,9 +399,10 @@ DEPENDENCIES sass sassc-rails selenium-webdriver - sentry-raven + sentry-rails + sentry-ruby simplecov - skylight + skylight (~> 5.0.0) spring spring-watcher-listen (~> 2.0.0) sprockets (= 3.7.2) @@ -432,7 +416,7 @@ DEPENDENCIES will_paginate-bootstrap4 (~> 0.2.2) RUBY VERSION - ruby 2.4.6p354 + ruby 2.7.6p219 BUNDLED WITH - 2.3.9 + 2.3.14 diff --git a/config/application.rb b/config/application.rb index 9d9526ab..e00a9e81 100644 --- a/config/application.rb +++ b/config/application.rb @@ -52,9 +52,21 @@ module Mapknitter # Version of your assets, change this if you want to expire all your assets config.assets.paths << Rails.root.join("public","lib") - Raven.configure do |config| - config.current_environment = ENV["COMPOSE_PROJECT_NAME"] || ENV["RAILS_ENV"] || %w(production) + Sentry.init do |config| + config.environment = ENV["COMPOSE_PROJECT_NAME"] || ENV["RAILS_ENV"] || %w(production) + config.enabled_environments = %w[production, mapknitter_stable, mapknitter_unstable] + config.breadcrumbs_logger = [:sentry_logger, :http_logger] + # To activate performance monitoring, set one of these options. + # We recommend adjusting the value in production: + config.traces_sample_rate = 0.5 + # use Rails' parameter filter to sanitize the event payload: + # for Rails 6+: + # filter = ActiveSupport::ParameterFilter.new(Rails.application.config.filter_parameters) + # for Rails 5: + filter = ActionDispatch::Http::ParameterFilter.new(Rails.application.config.filter_parameters) + config.before_send = lambda do |event, hint| + filter.filter(event.to_hash) + end end - end end diff --git a/dockerfiles/development b/dockerfiles/development index 20dd15d8..400a7c7a 100644 --- a/dockerfiles/development +++ b/dockerfiles/development @@ -2,34 +2,31 @@ # https://github.com/publiclab/mapknitter/ # This image deploys Mapknitter! -FROM ruby:2.4.6-stretch +FROM ruby:2.7.6 # Set correct environment variables. ENV HOME /root -# Backported GDAL -RUN echo "deb [trusted=yes] http://packages.laboratoriopublico.org/publiclab/ stretch main" > /etc/apt/sources.list.d/publiclab.list - # We bring our own key to verify our packages COPY sysadmin.publiclab.key /app/sysadmin.publiclab.key RUN apt-key add /app/sysadmin.publiclab.key > /dev/null 2>&1 # Install dependencies for Mapknitter RUN apt-get update -qq && apt-get install -y \ - nodejs gdal-bin curl procps git imagemagick python-gdal zip + nodejs curl procps git imagemagick -# Install dependencies for system tests -RUN apt-get -y install fonts-liberation libappindicator3-1 libasound2 \ - libatk-bridge2.0-0 libatspi2.0-0 libgtk-3-0 libnspr4 \ - libnss3 libx11-xcb1 libxss1 libxtst6 lsb-release xdg-utils && \ - wget https://github.com/webnicer/chrome-downloads/raw/master/x64.deb/google-chrome-stable_75.0.3770.142-1_amd64.deb \ - -O google-chrome.deb && \ - dpkg -i google-chrome.deb && \ - apt-get -fy install && \ - wget https://chromedriver.storage.googleapis.com/74.0.3729.6/chromedriver_linux64.zip && \ - unzip chromedriver_linux64.zip && \ - mv chromedriver /usr/local/bin/chromedriver && \ - chmod +x /usr/local/bin/chromedriver +# # Install dependencies for system tests +# RUN apt-get -y install fonts-liberation libayatana-appindicator libasound2 \ +# libatk-bridge2.0-0 libatspi2.0-0 libgtk-3-0 libnspr4 \ +# libnss3 libx11-xcb1 libxss1 libxtst6 lsb-release xdg-utils && \ +# wget https://github.com/webnicer/chrome-downloads/raw/master/x64.deb/google-chrome-stable_75.0.3770.142-1_amd64.deb \ +# -O google-chrome.deb && \ +# dpkg -i google-chrome.deb && \ +# apt-get -fy install && \ +# wget https://chromedriver.storage.googleapis.com/74.0.3729.6/chromedriver_linux64.zip && \ +# unzip chromedriver_linux64.zip && \ +# mv chromedriver /usr/local/bin/chromedriver && \ +# chmod +x /usr/local/bin/chromedriver # Configure ImageMagick COPY ./nolimit.xml /etc/ImageMagick-6/policy.xml diff --git a/dockerfiles/production b/dockerfiles/production index a5069d36..04d97bce 100644 --- a/dockerfiles/production +++ b/dockerfiles/production @@ -2,21 +2,18 @@ # https://github.com/publiclab/mapknitter/ # This image deploys Mapknitter! -FROM ruby:2.4.6-stretch +FROM ruby:2.7.6 # Set correct environment variables. ENV HOME /root -# Backported GDAL -RUN echo "deb [trusted=yes] http://packages.laboratoriopublico.org/publiclab/ stretch main" > /etc/apt/sources.list.d/publiclab.list - # We bring our own key to verify our packages COPY sysadmin.publiclab.key /app/sysadmin.publiclab.key RUN apt-key add /app/sysadmin.publiclab.key > /dev/null 2>&1 # Install dependencies for Mapknitter RUN apt-get update -qq && apt-get install --allow-unauthenticated -y --no-install-recommends \ - nodejs gdal-bin curl procps git imagemagick python-gdal zip + nodejs curl procps git imagemagick # Configure ImageMagick COPY ./nolimit.xml /etc/ImageMagick-6/policy.xml diff --git a/install_cloud.sh b/install_cloud.sh deleted file mode 100755 index c7ae85ae..00000000 --- a/install_cloud.sh +++ /dev/null @@ -1,28 +0,0 @@ -git checkout -b main -git pull -f origin main -gpg --keyserver hkp://pool.sks-keyservers.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB -curl -sSL https://get.rvm.io | bash -source ~/.rvm/scripts/rvm -rvm install ruby-2.4.6 -source $(rvm 2.4.6 do rvm env --path) -rvm use ruby-2.4.6 -gem install rails -v 4.2.11.1 -sudo apt-get update -sudo DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server -sudo apt-get -y install bundler libmysqlclient-dev imagemagick ruby-rmagick libfreeimage3 libfreeimage-dev ruby-dev libmagickcore-dev libmagickwand-dev npm nodejs-legacy -# exporter-only: -#sudo apt-get install gdal-bin python-gdal curl libcurl4-openssl-dev libssl-dev zip -gem install bundler -bundle install -cp db/schema.rb.example db/schema.rb -cp config/database.yml.cloud9.example config/database.yml -cp config/config.yml.example config/config.yml -sudo service mysql start -bin/rake db:migrate RAILS_ENV=development -rake db:setup -curl -sL https://deb.nodesource.com/setup_10.x | sudo -E bash - -sudo apt-get install -y nodejs -sudo apt-get update && sudo apt-get install yarn -sudo npm install -g yarn -sudo yarn install -echo "Done! Run the application with 'rails server -b 0.0.0.0'" diff --git a/lib/exporter-deps.sh b/lib/exporter-deps.sh index 453ceceb..6f7b6f56 100755 --- a/lib/exporter-deps.sh +++ b/lib/exporter-deps.sh @@ -2,12 +2,9 @@ sudo add-apt-repository -y ppa:ubuntugis/ppa sudo apt-get update # Installing Mapknitter dependencies -sudo apt-get install -y gdal-bin \ - python3-gdal \ - python-gdal \ - curl \ +sudo apt-get install -y curl \ libcurl4-openssl-dev \ - libssl-dev zip \ + libssl-dev \ libmysqlclient-dev \ imagemagick \ ruby-rmagick \ @@ -19,7 +16,7 @@ sudo apt-get install -y gdal-bin \ # Installing system tests dependencies sudo apt-get install -y fonts-liberation \ - libappindicator3-1 \ + libayatana-appindicator \ libasound2 \ libatk-bridge2.0-0 \ libatspi2.0-0 \ diff --git a/lib/exporter.rb b/lib/exporter.rb deleted file mode 100644 index 72dfa959..00000000 --- a/lib/exporter.rb +++ /dev/null @@ -1,367 +0,0 @@ -class Exporter - - def self.ulimit - # use ulimit to restrict to 7200 CPU seconds and 5gb virtual memory, and 5gB file storage: - #"ulimit -t 7200 && ulimit -v 5000000 && ulimit -f 5000000 && " - "ulimit -t 14400 && ulimit -v 5000000 && ulimit -f 10000000 && nice -n 19 " - end - - def self.get_working_directory(path) - "public/warps/" + path + "-working/" - end - - def self.warps_directory(path) - "public/warps/" + path + "/" - end - - def self.delete_temp_files(path) - system('rm -r ' + get_working_directory(path)) - system('rm ' + warps_directory(path) + '*.png') - end - - ######################## - ## Run on each image: - - # pixels per meter = pxperm - def self.generate_perspectival_distort(pxperm, path, nodes_array, id, image_file_name, image, height, width) - require 'net/http' - - # everything in -working/ can be deleted; - # this is just so we can use the files locally outside of s3 - working_directory = get_working_directory(path) - Dir.mkdir(working_directory) unless (File.exists?(working_directory) && File.directory?(working_directory)) - local_location = working_directory+id.to_s+'-'+image_file_name.to_s - - directory = warps_directory(path) - Dir.mkdir(directory) unless (File.exists?(directory) && File.directory?(directory)) - completed_local_location = directory+id.to_s+'.png' - - # everything -masked.png can be deleted - masked_local_location = directory+id.to_s+'-masked.png' - # everything -mask.png can be deleted - mask_location = directory+id.to_s+'-mask.png' - #completed_local_location = directory+id.to_s+'.tif' - # know everything -unwarped can be deleted - geotiff_location = directory+id.to_s+'-geo-unwarped.tif' - # everything -geo WITH AN ID could be deleted, but there is a feature request to preserve these - warped_geotiff_location = directory+id.to_s+'-geo.tif' - - northmost = nodes_array.first.lat - southmost = nodes_array.first.lat - westmost = nodes_array.first.lon - eastmost = nodes_array.first.lon - - nodes_array.each do |node| - northmost = node.lat if node.lat > northmost - southmost = node.lat if node.lat < southmost - westmost = node.lon if node.lon < westmost - eastmost = node.lon if node.lon > eastmost - end - - # puts northmost.to_s+','+southmost.to_s+','+westmost.to_s+','+eastmost.to_s - - scale = 20037508.34 - y1 = pxperm*Cartagen.spherical_mercator_lat_to_y(northmost,scale) - x1 = pxperm*Cartagen.spherical_mercator_lon_to_x(westmost,scale) - y2 = pxperm*Cartagen.spherical_mercator_lat_to_y(southmost,scale) - x2 = pxperm*Cartagen.spherical_mercator_lon_to_x(eastmost,scale) - # puts x1.to_s+','+y1.to_s+','+x2.to_s+','+y2.to_s - - # should determine if it's stored in s3 or locally: - if (image.url[0..3] == 'http') - Net::HTTP.start('s3.amazonaws.com') { |http| - #Net::HTTP.start('localhost') { |http| - puts (image.url) - resp = http.get(image.url) - open(local_location, "wb") { |file| - file.write(resp.body) - } - } - else - require "fileutils" - FileUtils.cp(Rails.root.to_s+'/public'+image.to_s,local_location) - end - - points = "" - maskpoints = "" - coordinates = "" - first = true - -#EXIF orientation values: -#Value 0th Row 0th Column -#1 top left side -#2 top right side -#3 bottom right side -#4 bottom left side -#5 left side top -#6 right side top -#7 right side bottom -#8 left side bottom - - rotation = (`identify -format %[exif:Orientation] #{local_location}`).to_i - #stdin, stdout, stderr = Open3.popen3('identify -format %[exif:Orientation] #{local_location}') - #rotation = stdout.readlines.first.to_s.to_i - #puts stderr.readlines - - if rotation == 6 - puts 'rotated CCW' - source_corners = source_corners = [[0,height],[0,0],[width,0],[width,height]] - elsif rotation == 8 - puts 'rotated CW' - source_corners = [[width,0],[width,height],[0,height],[0,0]] - elsif rotation == 3 - puts 'rotated 180 deg' - source_corners = [[width,height],[0,height],[0,0],[width,0]] - else - source_corners = [[0,0],[width,0],[width,height],[0,height]] - end - - maxdimension = 0 - - nodes_array.each do |node| - corner = source_corners.shift - nx1 = corner[0] - ny1 = corner[1] - nx2 = -x1+(pxperm*Cartagen.spherical_mercator_lon_to_x(node.lon,scale)) - ny2 = y1-(pxperm*Cartagen.spherical_mercator_lat_to_y(node.lat,scale)) - - points = points + ' ' unless first - maskpoints = maskpoints + ' ' unless first - points = points + nx1.to_s + ',' + ny1.to_s + ' ' + nx2.to_i.to_s + ',' + ny2.to_i.to_s - maskpoints = maskpoints + nx2.to_i.to_s + ',' + ny2.to_i.to_s - first = false - # we need to find an origin; find northwestern-most point - coordinates = coordinates+' -gcp '+nx2.to_s+', '+ny2.to_s+', '+node.lon.to_s + ', ' + node.lat.to_s - - # identify largest dimension to set canvas size for ImageMagick: - maxdimension = nx1.to_i if maxdimension < nx1.to_i - maxdimension = ny1.to_i if maxdimension < ny1.to_i - maxdimension = nx2.to_i if maxdimension < nx2.to_i - maxdimension = ny2.to_i if maxdimension < ny2.to_i - end - - # close mask polygon: - maskpoints = maskpoints + ' ' - nx2 = -x1+(pxperm*Cartagen.spherical_mercator_lon_to_x(nodes_array.first.lon,scale)) - ny2 = y1-(pxperm*Cartagen.spherical_mercator_lat_to_y(nodes_array.first.lat,scale)) - maskpoints = maskpoints + nx2.to_i.to_s + ',' + ny2.to_i.to_s - - height = (y1-y2).to_i.to_s - width = (-x1+x2).to_i.to_s - - # http://www.imagemagick.org/discourse-server/viewtopic.php?f=1&t=11319 - # http://www.imagemagick.org/discourse-server/viewtopic.php?f=3&t=8764 - # read about equalization - # -equalize - # -contrast-stretch 0 - - imageMagick = "convert " - imageMagick += "-contrast-stretch 0 " - imageMagick += local_location+" " - imageMagick += "-crop "+maxdimension.to_i.to_s+"x"+maxdimension.to_i.to_s+"+0+0! " - imageMagick += "-flatten " - imageMagick += "-distort Perspective '"+points+"' " - imageMagick += "-flatten " - if width > height - imageMagick += "-crop "+width+"x"+width+"+0+0\! " - else - imageMagick += "-crop "+height+"x"+height+"+0+0\! " - end - imageMagick += "+repage " - imageMagick += completed_local_location - puts imageMagick - system(self.ulimit+imageMagick) - - # create a mask (later we can blur edges here) - imageMagick2 = 'convert +antialias ' - if width > height - imageMagick2 += "-size "+width+"x"+width+" " - else - imageMagick2 += "-size "+height+"x"+height+" " - end - # attempt at blurred edges in masking, but I've given up, as gdal_merge doesn't seem to respect variable-opacity alpha channels - imageMagick2 += ' xc:none -draw "fill black stroke red stroke-width 30 polyline ' - imageMagick2 += maskpoints + '" ' - imageMagick2 += ' -alpha set -channel A -transparent red -blur 0x8 -channel R -evaluate set 0 +channel '+mask_location - #imageMagick2 += ' xc:none -draw "fill black stroke none polyline ' - #imageMagick2 += maskpoints + '" ' - #imageMagick2 += ' '+mask_location - puts imageMagick2 - system(self.ulimit+imageMagick2) - - imageMagick3 = 'composite '+mask_location+' '+completed_local_location+' -compose DstIn -alpha Set '+masked_local_location - puts imageMagick3 - system(self.ulimit+imageMagick3) - - gdal_translate = "gdal_translate -of GTiff -a_srs EPSG:4326 "+coordinates+' -co "TILED=NO" '+masked_local_location+' '+geotiff_location - puts gdal_translate - system(self.ulimit+gdal_translate) - - #gdalwarp = 'gdalwarp -srcnodata "255" -dstnodata 0 -cblend 30 -of GTiff -t_srs EPSG:4326 '+geotiff_location+' '+warped_geotiff_location - gdalwarp = 'gdalwarp -of GTiff -t_srs EPSG:4326 '+geotiff_location+' '+warped_geotiff_location - puts gdalwarp - system(self.ulimit+gdalwarp) - - # deletions could happen here; do it in distinct method so we can run it independently - delete_temp_files(path) - - [x1,y1] - end - - ######################## - ## Run on maps: - - # distort all warpables, returns upper left corner coords in x,y - def self.distort_warpables(scale, warpables, export, slug) - - puts '> generating geotiffs of each warpable in GDAL' - lowest_x=0 - lowest_y=0 - warpable_coords = [] - current = 0 - warpables.each do |warpable| - current += 1 - - ## TODO: refactor to generate static status file: - export.status = 'warping '+current.to_s+' of '+warpables.length.to_s - puts 'warping '+current.to_s+' of '+warpables.length.to_s - export.save - ## - - my_warpable_coords = warpable.generate_perspectival_distort(scale,slug) - puts '- '+my_warpable_coords.to_s - warpable_coords << my_warpable_coords - lowest_x = my_warpable_coords.first if (my_warpable_coords.first < lowest_x || lowest_x.zero? ) - lowest_y = my_warpable_coords.last if (my_warpable_coords.last < lowest_y || lowest_y.zero? ) - end - [lowest_x,lowest_y,warpable_coords] - end - - # generate a tiff from all warpable images in this set - def self.generate_composite_tiff(coords, origin, placed_warpables, slug, ordered) - directory = "public/warps/"+slug+"/" - composite_location = directory+slug+'-geo.tif' - geotiffs = '' - minlat = nil - minlon = nil - maxlat = nil - maxlon = nil - placed_warpables.each do |warpable| - warpable.nodes_array.each do |n| - minlat = n.lat if minlat == nil || n.lat < minlat - minlon = n.lon if minlon == nil || n.lon < minlon - maxlat = n.lat if maxlat == nil || n.lat > maxlat - maxlon = n.lon if maxlon == nil || n.lon > maxlon - end - end - if ordered != true - # sort by area; this would be overridden by a provided order - warpables = placed_warpables.sort{|a,b|b.poly_area <=> a.poly_area} - end - warpables.each do |warpable| - geotiffs += ' '+directory+warpable.id.to_s+'-geo.tif' - end - gdalwarp = "gdalwarp -s_srs EPSG:3857 -te #{minlon} #{minlat} #{maxlon} #{maxlat} #{geotiffs} #{directory}#{slug}-geo.tif" - puts gdalwarp - system(self.ulimit+gdalwarp) - composite_location - end - - # generates a tileset at root/public/tms// - # root is something like https://mapknitter.org - def self.generate_tiles(key, slug, root) - key = "AIzaSyAOLUQngEmJv0_zcG1xkGq-CXIPpLQY8iQ" if key.blank? - gdal2tiles = 'gdal2tiles.py -k --s_srs EPSG:4326 -t "'+slug+'" -g "'+key+'" '+root+'/public/warps/'+slug+'/'+slug+'-geo.tif '+root+'/public/tms/'+slug+"/" - puts gdal2tiles - system(self.ulimit+gdal2tiles) - end - - # zips up tiles at root/public/tms/.zip; - def self.zip_tiles(slug) - rmzip = 'cd public/tms/ && rm '+slug+'.zip && cd ../../' - system(rmzip) - zip = 'cd public/tms/ && ' + self.ulimit + 'zip -rq '+slug+'.zip '+slug+'/ && cd ../../' - system(zip) - end - - # generates a tileset at root/public/tms// - def self.generate_jpg(slug, root) - imageMagick = 'convert -background white -flatten '+root+'/public/warps/'+slug+'/'+slug+'-geo.tif '+root+'/public/warps/'+slug+'/'+slug+'.jpg' - system(self.ulimit+imageMagick) - end - - # runs the above map functions while maintaining a record of state in an Export model; - # we'll be replacing the export model state with a flat status file - def self.run_export(user,resolution,export,id,slug,root,average_scale,placed_warpables,key) - begin - export.user_id = user.id if user - export.status = 'starting' - export.tms = false - export.geotiff = false - export.zip = false - export.jpg = false - export.bands_string = 'default bands_string' - export.save - - directory = "#{root}/public/warps/"+slug+"/" - stdin, stdout, stderr = Open3.popen3('rm -r '+directory.to_s) - puts stdout.readlines - puts stderr.readlines - stdin, stdout, stderr = Open3.popen3("rm -r #{root}/public/tms/#{slug}") - puts stdout.readlines - puts stderr.readlines - - puts '> averaging scales; resolution: ' + resolution.to_s - pxperm = 100/(resolution).to_f || average_scale # pixels per meter - puts '> scale: ' + pxperm.to_s + 'pxperm' - - puts '> distorting warpables' - - origin = self.distort_warpables(pxperm, placed_warpables, export, slug) - warpable_coords = origin.pop - - export.status = 'compositing' - export.save - - puts '> generating composite tiff' - composite_location = self.generate_composite_tiff(warpable_coords,origin,placed_warpables,slug,false) # no ordering yet - - info = (`identify -quiet -format '%b,%w,%h' #{composite_location}`).split(',') - puts info - - if info[0] != '' - export.geotiff = true - export.size = info[0] - export.width = info[1] - export.height = info[2] - export.cm_per_pixel = 100.0000/pxperm - export.status = 'tiling' - export.save - end - - puts '> generating tiles' - export.tms = true if self.generate_tiles(key, slug, root) - export.status = 'zipping tiles' - export.save - - puts '> zipping tiles' - export.zip = true if self.zip_tiles(slug) - export.status = 'creating jpg' - export.save - - puts '> generating jpg' - export.jpg = true if self.generate_jpg(slug, root) - export.status = 'complete' - export.save - - rescue SystemCallError => e - puts "ERROR" - puts e.inspect - puts "END ERROR" - export.status = 'failed' - export.save - end - return export.status - end - -end diff --git a/test/controllers/export_controller_test.rb b/test/controllers/export_controller_test.rb index 1e4600f2..213dbb25 100644 --- a/test/controllers/export_controller_test.rb +++ b/test/controllers/export_controller_test.rb @@ -22,87 +22,10 @@ class ExportControllerTest < ActionController::TestCase assert_equal url, Export.last.export_url end - test 'should create jpg after export' do - map = maps(:cubbon) - system('mkdir -p public/warps/cubbon-park') - system('cp test/fixtures/demo.png public/warps/cubbon-park/cubbon-park.jpg') - - get :jpg, params: { id: map.slug} - assert_response :success - assert_includes 'image/jpeg', response.content_type - end - - test 'should create geotiff after export' do - map = maps(:cubbon) - system('mkdir -p public/warps/cubbon-park') - system('cp test/fixtures/demo.png public/warps/cubbon-park/cubbon-park-geo.tif') - - get :geotiff, params: { id: map.slug} - assert_response :success - assert_includes 'image/tiff', response.content_type - end - - test 'should not cancel if not logged in' do - get :cancel, params: { id: @map.id} - assert_response :success - assert_equal 'You must be logged in to export, unless the map is anonymous.', @response.body - assert assigns[:map] - assert_equal 'text/plain', @response.content_type - assert flash.empty? - end - - test 'should cancel export' do - session[:user_id] = 1 - get :cancel, params: { id: @map.id} - assert_response :success - assert_equal 'cancelled', @response.body - assert assigns[:map] - end - - test 'should redirect after cancelling' do - session[:user_id] = 1 - get :cancel, params: { id: @map.id, exports: 'cess'} - assert_response :redirect - assert flash.present? - assert_redirected_to '/exports' - end - - test 'should display export progress' do - get :progress, params: { id: @map.id} - assert_response :success - assert_equal 'export not running', @response.body - assert_equal 'text/plain', @response.content_type - end - - test 'should display progress with no export' do - get :progress, params: { id: 4} - assert_response :success - assert_equal 'export has not been run', @response.body - end - - test 'should display progress completed' do - get :progress, params: { id: 2} - assert_response :success - assert_equal 'complete', @response.body - end - - test 'should display progress failed' do - get :progress, params: { id: 3} - assert_response :success - assert_equal 'export failed', @response.body - end - # does not test the exporter client test 'should display export status' do session[:user_id] = 1 get :status, params: { id: @map.id} assert_response :success end - - test 'should display error if no export' do - session[:user_id] = 1 - get :status, params: { id: 4} - assert_response :success - # assert_equal { status: 'export has not been run' }.to_json, @response.body - end end diff --git a/test/models/export_test.rb b/test/models/export_test.rb deleted file mode 100644 index 8b50569f..00000000 --- a/test/models/export_test.rb +++ /dev/null @@ -1,21 +0,0 @@ -require 'test_helper' - -class ExportTest < ActiveSupport::TestCase - test 'should count methods' do - average = Export.all.map(&:cm_per_pixel).sum/Export.count - assert_not_nil Export.average_cm_per_pixel - assert_equal average, Export.average_cm_per_pixel - assert_not_nil Export.histogram_cm_per_pixel - assert_not_nil Export.histogram_cm_per_pixel_in_tens - assert_not_nil Export.export_count - assert_not_nil Export.exporting - - Export.delete_all - assert_empty Export.histogram_cm_per_pixel - assert_equal 0, Export.average_cm_per_pixel - end - - test 'should export running' do - assert !Export.last.running? - end -end diff --git a/test/models/exporter_test.rb b/test/models/exporter_test.rb deleted file mode 100644 index ffab0894..00000000 --- a/test/models/exporter_test.rb +++ /dev/null @@ -1,57 +0,0 @@ -require 'test_helper' - -class ExporterTest < ActiveSupport::TestCase - test 'should export warpable using isolated exporter lib' do - - # make a sample image - system('mkdir -p public/system/images/1/original') - system('cp test/fixtures/demo.png public/system/images/1/original/') - system('mkdir -p public/warps/saugus-landfill-incinerator') - system('mkdir -p public/tms/saugus-landfill-incinerator') - system('touch public/warps/saugus-landfill-incinerator/folder') - assert File.exist?('public/warps/saugus-landfill-incinerator/folder') - - scale = 2 - - w = warpables(:one) - coords = Exporter.generate_perspectival_distort(scale, w.map.slug, w.nodes_array, w.id, w.image_file_name, w.image, w.height, w.width) - assert coords - assert Exporter.get_working_directory(w.map.slug) - assert Exporter.warps_directory(w.map.slug) - - map = maps(:saugus) - - # get rid of existing geotiff - system('rm -r public/warps/saugus-landfill-incinerator/1-geo.tif') - # make a sample image - system('mkdir -p public/system/images/2/original/') - system('cp test/fixtures/demo.png public/system/images/2/original/test.png') - origin = Exporter.distort_warpables(scale, map.warpables, map.export, map.slug) - lowest_x, lowest_y, warpable_coords = origin - assert origin - ordered = false - - system('mkdir -p public/warps/saugus-landfill-incinerator') - system('mkdir -p public/tms/saugus-landfill-incinerator') - # these params could be compressed - warpable coords is part of origin; are coords and origin required? - assert Exporter.generate_composite_tiff(warpable_coords, origin, map.placed_warpables, map.slug, ordered) - assert Exporter.generate_tiles('', map.slug, Rails.root.to_s) - assert Exporter.zip_tiles(map.slug) - assert Exporter.generate_jpg(map.slug, Rails.root.to_s) - resolution = 20 - assert Exporter.run_export(User.last, resolution, map.export, map.id, map.slug, Rails.root.to_s, map.average_scale, map.placed_warpables, '') - - # test deletion of the files; they were already deleted in run_export, so let's make sample ones: - # make a sample image - system('mkdir -p public/system/images/2/original/') - system('touch public/system/images/2/original/test.png') - system('mkdir -p public/warps/saugus-landfill-incinerator') - system('mkdir -p public/tms/saugus-landfill-incinerator') - system('touch public/warps/saugus-landfill-incinerator/folder') - assert File.exist?('public/warps/saugus-landfill-incinerator/folder') - system('mkdir -p public/warps/saugus-landfill-incinerator-working') - system('touch public/warps/saugus-landfill-incinerator/test.png') - assert Exporter.delete_temp_files(w.map.slug) - end -end - diff --git a/test/models/map_test.rb b/test/models/map_test.rb index e550201c..5ccac60f 100644 --- a/test/models/map_test.rb +++ b/test/models/map_test.rb @@ -49,42 +49,6 @@ class MapTest < ActiveSupport::TestCase assert_not_nil map.latest_export assert_not_nil map.nodes assert_not_nil map.average_cm_per_pixel - - # use a map fixture with no warpables - village = maps(:village) - assert_equal 0, village.average_cm_per_pixel - - resolution = 20 - assert_not_nil map.run_export(users(:quentin), resolution) #map.average_cm_per_pixel) - - # main issue will be that it creates and continuously updates an Export model. - # we could shift this to a polling model, either on the client side (eliminating the Export model) - # ... or some other way to make it possible to do many of these tasks without needing ActiveRecord - - # let's start at the bottom and factor this all out working upwards - - # refactor so that we pass in as much in parameters as possible, reducing in-model cross-references - - # creates an Export and sets initial values - # depends on: map.average_scale - # runs self.distort_warpables(pxperm) - # runs self.generate_composite_tiff(warpable_coords,origin) - # runs `identify` and assigns some values (height, width) to Export - # runs export.tms = true if self.generate_tiles - # runs export.zip = true if self.zip_tiles - # runs export.jpg = true if self.generate_jpg - # map.distort_warpables(scale) - # collects self.placed_warpables - # runs on each one: warpable.generate_perspectival_distort(scale,self.slug) - # map.generate_composite_tiff(coords,origin) - # collects self.placed_warpables - # runs gdal_warp on the output of each, flattening onto a single geotiff - # map.generate_tiles - # runs on composite tiff output: gdal2tiles = 'gdal2tiles.py -k -t "'+self.slug+'" -g "'+google_api_key+'" '+Rails.root.to_s+'/public/warps/'+self.slug+'/'+self.slug+'-geo.tif '+Rails.root.to_s+'/public/tms/'+self.slug+"/" - # map.zip_tiles - # map.generate_jpg - # runs convert on composite tiff - end test 'should have histograms' do @@ -94,7 +58,6 @@ class MapTest < ActiveSupport::TestCase assert_not_nil hist assert_not_nil map.grouped_images_histogram(3) assert_equal hist.count/3, map.grouped_images_histogram(3).count - end test 'should have nearby maps' do diff --git a/test/models/warpable_test.rb b/test/models/warpable_test.rb index e1d25184..69901404 100644 --- a/test/models/warpable_test.rb +++ b/test/models/warpable_test.rb @@ -53,33 +53,4 @@ class WarpableTest < ActiveSupport::TestCase Warpable.delete_all assert_empty Warpable.histogram_cm_per_pixel end - - test 'should try export warpables' do - # make a sample image - system('mkdir -p public/warps/saugus-landfill-incinerator-working') - system('mkdir -p public/system/images/1/original') - system('cp test/fixtures/demo.png public/system/images/1/original/') - system('mkdir -p public/warps/saugus-landfill-incinerator') - system('touch public/warps/saugus-landfill-incinerator/folder') - system('mkdir -p public/system/images/2/original/') - system('cp test/fixtures/demo.png public/system/images/2/original/test.png') - system('cp test/fixtures/demo.png public/warps/saugus-landfill-incinerator/1-geo.tif') - assert File.exist?('public/warps/saugus-landfill-incinerator/folder') - - origin = Exporter.distort_warpables(2, - @map.warpables, - @map.export, - @map.slug) - - assert_not_nil @warp.save_dimensions - assert_not_nil @warp.user_id - - Exporter.generate_composite_tiff(nil, - origin, - @map.placed_warpables, - @map.slug, - false) - - assert File.exist?('public/warps/saugus-landfill-incinerator/1-geo.tif') - end end