mirror of
https://github.com/publiclab/mapknitter.git
synced 2025-12-05 16:00:00 +01:00
migrate sentry-raven to sentry-ruby, upgrade to Ruby 2.7.6, remove GDAL 😲 (#1712)
* update application.rb * try to complete migration * fixed gemfile.lock * updated to Ruby 2.7.3 * more ruby 2.7.3 updates for dockerfiles * dockerfiles to 2.7.6 * remove local map exporting and gdal * more gdal removal * libappindicator3-7 \ * fixed libindicator3-7 * update for Debian 10 - libayatana-appindicator * all ruby 2.7.6 now * libayatana-appindicator in dockerfiles * remove system test deps for dev dockerfile
This commit is contained in:
Jeffrey Warren
GitHub
16
.github/workflows/tests.yml
vendored
16
.github/workflows/tests.yml
vendored
@@ -13,7 +13,7 @@ jobs:
|
||||
- name: Prepare Ruby
|
||||
uses: ruby/setup-ruby@v1
|
||||
with:
|
||||
ruby-version: 2.4.6
|
||||
ruby-version: 2.7.6
|
||||
bundler-cache: true
|
||||
- name: Rubocop report
|
||||
env:
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: ruby/setup-ruby@v1
|
||||
with:
|
||||
ruby-version: 2.4.6
|
||||
ruby-version: 2.7.6
|
||||
bundler-cache: true
|
||||
- uses: ./.github/actions/setup-test-environment
|
||||
- uses: egordm/gha-yarn-node-cache@v1
|
||||
@@ -47,7 +47,7 @@ jobs:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: ruby/setup-ruby@v1
|
||||
with:
|
||||
ruby-version: 2.4.6
|
||||
ruby-version: 2.7.6
|
||||
bundler-cache: true
|
||||
- uses: ./.github/actions/setup-test-environment
|
||||
- uses: egordm/gha-yarn-node-cache@v1
|
||||
@@ -67,7 +67,7 @@ jobs:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: ruby/setup-ruby@v1
|
||||
with:
|
||||
ruby-version: 2.4.6
|
||||
ruby-version: 2.7.6
|
||||
bundler-cache: true
|
||||
- uses: ./.github/actions/setup-test-environment
|
||||
- uses: egordm/gha-yarn-node-cache@v1
|
||||
@@ -99,7 +99,7 @@ jobs:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: ruby/setup-ruby@v1
|
||||
with:
|
||||
ruby-version: 2.4.6
|
||||
ruby-version: 2.7.6
|
||||
bundler-cache: true
|
||||
- uses: nanasess/setup-chromedriver@v1.0.7
|
||||
- uses: ./.github/actions/setup-test-environment
|
||||
@@ -131,7 +131,7 @@ jobs:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: ruby/setup-ruby@v1
|
||||
with:
|
||||
ruby-version: 2.4.6
|
||||
ruby-version: 2.7.6
|
||||
bundler-cache: true
|
||||
- uses: ./.github/actions/setup-test-environment
|
||||
- name: 'Development Docker Build'
|
||||
@@ -143,7 +143,7 @@ jobs:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: ruby/setup-ruby@v1
|
||||
with:
|
||||
ruby-version: 2.4.6
|
||||
ruby-version: 2.7.6
|
||||
bundler-cache: true
|
||||
- uses: ./.github/actions/setup-test-environment
|
||||
- name: 'Development Docker Build'
|
||||
@@ -155,7 +155,7 @@ jobs:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: ruby/setup-ruby@v1
|
||||
with:
|
||||
ruby-version: 2.4.6
|
||||
ruby-version: 2.7.6
|
||||
bundler-cache: true
|
||||
- uses: ./.github/actions/setup-test-environment
|
||||
- uses: egordm/gha-yarn-node-cache@v1
|
||||
|
||||
9
Gemfile
9
Gemfile
@@ -1,11 +1,12 @@
|
||||
source 'https://rubygems.org'
|
||||
|
||||
ruby '2.4.6'
|
||||
ruby '2.7.6'
|
||||
gem 'rails', '~> 5.2.8'
|
||||
gem 'rake', '~> 13.0.1'
|
||||
gem 'tzinfo-data'
|
||||
gem 'skylight'
|
||||
gem 'sentry-raven'
|
||||
gem 'skylight', '~> 5.0.0'
|
||||
gem 'sentry-ruby'
|
||||
gem 'sentry-rails'
|
||||
|
||||
gem 'will_paginate', '3.3.1'
|
||||
gem 'will_paginate-bootstrap4', '~> 0.2.2'
|
||||
@@ -97,4 +98,4 @@ group :passenger do
|
||||
gem 'passenger'
|
||||
end
|
||||
|
||||
gem 'httparty'
|
||||
gem 'httparty'
|
||||
46
Gemfile.lock
46
Gemfile.lock
@@ -98,25 +98,6 @@ GEM
|
||||
execjs (2.8.1)
|
||||
faker (2.12.0)
|
||||
i18n (>= 1.6, < 2)
|
||||
faraday (1.7.1)
|
||||
faraday-em_http (~> 1.0)
|
||||
faraday-em_synchrony (~> 1.0)
|
||||
faraday-excon (~> 1.1)
|
||||
faraday-httpclient (~> 1.0.1)
|
||||
faraday-net_http (~> 1.0)
|
||||
faraday-net_http_persistent (~> 1.1)
|
||||
faraday-patron (~> 1.0)
|
||||
faraday-rack (~> 1.0)
|
||||
multipart-post (>= 1.2, < 3)
|
||||
ruby2_keywords (>= 0.0.4)
|
||||
faraday-em_http (1.0.0)
|
||||
faraday-em_synchrony (1.0.0)
|
||||
faraday-excon (1.1.0)
|
||||
faraday-httpclient (1.0.1)
|
||||
faraday-net_http (1.0.1)
|
||||
faraday-net_http_persistent (1.2.0)
|
||||
faraday-patron (1.0.0)
|
||||
faraday-rack (1.0.0)
|
||||
ffi (1.15.5)
|
||||
friendly_id (5.4.2)
|
||||
activerecord (>= 4.0.0)
|
||||
@@ -174,7 +155,6 @@ GEM
|
||||
msgpack (1.4.5)
|
||||
multi_json (1.15.0)
|
||||
multi_xml (0.6.0)
|
||||
multipart-post (2.1.1)
|
||||
mysql2 (0.5.4)
|
||||
net-http-persistent (2.9.4)
|
||||
nio4r (2.5.8)
|
||||
@@ -296,7 +276,6 @@ GEM
|
||||
ruby-openid (>= 2.1.7)
|
||||
ruby-prof (1.4.2)
|
||||
ruby-progressbar (1.11.0)
|
||||
ruby2_keywords (0.0.5)
|
||||
rubyzip (2.3.2)
|
||||
sass (3.7.4)
|
||||
sass-listen (~> 4.0.0)
|
||||
@@ -314,16 +293,20 @@ GEM
|
||||
selenium-webdriver (3.142.7)
|
||||
childprocess (>= 0.5, < 4.0)
|
||||
rubyzip (>= 1.2.2)
|
||||
sentry-raven (3.1.2)
|
||||
faraday (>= 1.0)
|
||||
sentry-rails (5.3.1)
|
||||
railties (>= 5.0)
|
||||
sentry-ruby-core (~> 5.3.1)
|
||||
sentry-ruby (5.3.1)
|
||||
concurrent-ruby (~> 1.0, >= 1.0.2)
|
||||
sentry-ruby-core (= 5.3.1)
|
||||
sentry-ruby-core (5.3.1)
|
||||
concurrent-ruby
|
||||
simplecov (0.18.5)
|
||||
docile (~> 1.1)
|
||||
simplecov-html (~> 0.11)
|
||||
simplecov-html (0.12.3)
|
||||
skylight (4.3.2)
|
||||
skylight-core (= 4.3.2)
|
||||
skylight-core (4.3.2)
|
||||
activesupport (>= 4.2.0)
|
||||
skylight (5.0.1)
|
||||
activesupport (>= 5.2.0)
|
||||
spring (2.1.1)
|
||||
spring-watcher-listen (2.0.1)
|
||||
listen (>= 2.7, < 4.0)
|
||||
@@ -416,9 +399,10 @@ DEPENDENCIES
|
||||
sass
|
||||
sassc-rails
|
||||
selenium-webdriver
|
||||
sentry-raven
|
||||
sentry-rails
|
||||
sentry-ruby
|
||||
simplecov
|
||||
skylight
|
||||
skylight (~> 5.0.0)
|
||||
spring
|
||||
spring-watcher-listen (~> 2.0.0)
|
||||
sprockets (= 3.7.2)
|
||||
@@ -432,7 +416,7 @@ DEPENDENCIES
|
||||
will_paginate-bootstrap4 (~> 0.2.2)
|
||||
|
||||
RUBY VERSION
|
||||
ruby 2.4.6p354
|
||||
ruby 2.7.6p219
|
||||
|
||||
BUNDLED WITH
|
||||
2.3.9
|
||||
2.3.14
|
||||
|
||||
@@ -52,9 +52,21 @@ module Mapknitter
|
||||
# Version of your assets, change this if you want to expire all your assets
|
||||
config.assets.paths << Rails.root.join("public","lib")
|
||||
|
||||
Raven.configure do |config|
|
||||
config.current_environment = ENV["COMPOSE_PROJECT_NAME"] || ENV["RAILS_ENV"] || %w(production)
|
||||
Sentry.init do |config|
|
||||
config.environment = ENV["COMPOSE_PROJECT_NAME"] || ENV["RAILS_ENV"] || %w(production)
|
||||
config.enabled_environments = %w[production, mapknitter_stable, mapknitter_unstable]
|
||||
config.breadcrumbs_logger = [:sentry_logger, :http_logger]
|
||||
# To activate performance monitoring, set one of these options.
|
||||
# We recommend adjusting the value in production:
|
||||
config.traces_sample_rate = 0.5
|
||||
# use Rails' parameter filter to sanitize the event payload:
|
||||
# for Rails 6+:
|
||||
# filter = ActiveSupport::ParameterFilter.new(Rails.application.config.filter_parameters)
|
||||
# for Rails 5:
|
||||
filter = ActionDispatch::Http::ParameterFilter.new(Rails.application.config.filter_parameters)
|
||||
config.before_send = lambda do |event, hint|
|
||||
filter.filter(event.to_hash)
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
|
||||
@@ -2,34 +2,31 @@
|
||||
# https://github.com/publiclab/mapknitter/
|
||||
# This image deploys Mapknitter!
|
||||
|
||||
FROM ruby:2.4.6-stretch
|
||||
FROM ruby:2.7.6
|
||||
|
||||
# Set correct environment variables.
|
||||
ENV HOME /root
|
||||
|
||||
# Backported GDAL
|
||||
RUN echo "deb [trusted=yes] http://packages.laboratoriopublico.org/publiclab/ stretch main" > /etc/apt/sources.list.d/publiclab.list
|
||||
|
||||
# We bring our own key to verify our packages
|
||||
COPY sysadmin.publiclab.key /app/sysadmin.publiclab.key
|
||||
RUN apt-key add /app/sysadmin.publiclab.key > /dev/null 2>&1
|
||||
|
||||
# Install dependencies for Mapknitter
|
||||
RUN apt-get update -qq && apt-get install -y \
|
||||
nodejs gdal-bin curl procps git imagemagick python-gdal zip
|
||||
nodejs curl procps git imagemagick
|
||||
|
||||
# Install dependencies for system tests
|
||||
RUN apt-get -y install fonts-liberation libappindicator3-1 libasound2 \
|
||||
libatk-bridge2.0-0 libatspi2.0-0 libgtk-3-0 libnspr4 \
|
||||
libnss3 libx11-xcb1 libxss1 libxtst6 lsb-release xdg-utils && \
|
||||
wget https://github.com/webnicer/chrome-downloads/raw/master/x64.deb/google-chrome-stable_75.0.3770.142-1_amd64.deb \
|
||||
-O google-chrome.deb && \
|
||||
dpkg -i google-chrome.deb && \
|
||||
apt-get -fy install && \
|
||||
wget https://chromedriver.storage.googleapis.com/74.0.3729.6/chromedriver_linux64.zip && \
|
||||
unzip chromedriver_linux64.zip && \
|
||||
mv chromedriver /usr/local/bin/chromedriver && \
|
||||
chmod +x /usr/local/bin/chromedriver
|
||||
# # Install dependencies for system tests
|
||||
# RUN apt-get -y install fonts-liberation libayatana-appindicator libasound2 \
|
||||
# libatk-bridge2.0-0 libatspi2.0-0 libgtk-3-0 libnspr4 \
|
||||
# libnss3 libx11-xcb1 libxss1 libxtst6 lsb-release xdg-utils && \
|
||||
# wget https://github.com/webnicer/chrome-downloads/raw/master/x64.deb/google-chrome-stable_75.0.3770.142-1_amd64.deb \
|
||||
# -O google-chrome.deb && \
|
||||
# dpkg -i google-chrome.deb && \
|
||||
# apt-get -fy install && \
|
||||
# wget https://chromedriver.storage.googleapis.com/74.0.3729.6/chromedriver_linux64.zip && \
|
||||
# unzip chromedriver_linux64.zip && \
|
||||
# mv chromedriver /usr/local/bin/chromedriver && \
|
||||
# chmod +x /usr/local/bin/chromedriver
|
||||
|
||||
# Configure ImageMagick
|
||||
COPY ./nolimit.xml /etc/ImageMagick-6/policy.xml
|
||||
|
||||
@@ -2,21 +2,18 @@
|
||||
# https://github.com/publiclab/mapknitter/
|
||||
# This image deploys Mapknitter!
|
||||
|
||||
FROM ruby:2.4.6-stretch
|
||||
FROM ruby:2.7.6
|
||||
|
||||
# Set correct environment variables.
|
||||
ENV HOME /root
|
||||
|
||||
# Backported GDAL
|
||||
RUN echo "deb [trusted=yes] http://packages.laboratoriopublico.org/publiclab/ stretch main" > /etc/apt/sources.list.d/publiclab.list
|
||||
|
||||
# We bring our own key to verify our packages
|
||||
COPY sysadmin.publiclab.key /app/sysadmin.publiclab.key
|
||||
RUN apt-key add /app/sysadmin.publiclab.key > /dev/null 2>&1
|
||||
|
||||
# Install dependencies for Mapknitter
|
||||
RUN apt-get update -qq && apt-get install --allow-unauthenticated -y --no-install-recommends \
|
||||
nodejs gdal-bin curl procps git imagemagick python-gdal zip
|
||||
nodejs curl procps git imagemagick
|
||||
|
||||
# Configure ImageMagick
|
||||
COPY ./nolimit.xml /etc/ImageMagick-6/policy.xml
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
git checkout -b main
|
||||
git pull -f origin main
|
||||
gpg --keyserver hkp://pool.sks-keyservers.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
|
||||
curl -sSL https://get.rvm.io | bash
|
||||
source ~/.rvm/scripts/rvm
|
||||
rvm install ruby-2.4.6
|
||||
source $(rvm 2.4.6 do rvm env --path)
|
||||
rvm use ruby-2.4.6
|
||||
gem install rails -v 4.2.11.1
|
||||
sudo apt-get update
|
||||
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server
|
||||
sudo apt-get -y install bundler libmysqlclient-dev imagemagick ruby-rmagick libfreeimage3 libfreeimage-dev ruby-dev libmagickcore-dev libmagickwand-dev npm nodejs-legacy
|
||||
# exporter-only:
|
||||
#sudo apt-get install gdal-bin python-gdal curl libcurl4-openssl-dev libssl-dev zip
|
||||
gem install bundler
|
||||
bundle install
|
||||
cp db/schema.rb.example db/schema.rb
|
||||
cp config/database.yml.cloud9.example config/database.yml
|
||||
cp config/config.yml.example config/config.yml
|
||||
sudo service mysql start
|
||||
bin/rake db:migrate RAILS_ENV=development
|
||||
rake db:setup
|
||||
curl -sL https://deb.nodesource.com/setup_10.x | sudo -E bash -
|
||||
sudo apt-get install -y nodejs
|
||||
sudo apt-get update && sudo apt-get install yarn
|
||||
sudo npm install -g yarn
|
||||
sudo yarn install
|
||||
echo "Done! Run the application with 'rails server -b 0.0.0.0'"
|
||||
@@ -2,12 +2,9 @@ sudo add-apt-repository -y ppa:ubuntugis/ppa
|
||||
sudo apt-get update
|
||||
|
||||
# Installing Mapknitter dependencies
|
||||
sudo apt-get install -y gdal-bin \
|
||||
python3-gdal \
|
||||
python-gdal \
|
||||
curl \
|
||||
sudo apt-get install -y curl \
|
||||
libcurl4-openssl-dev \
|
||||
libssl-dev zip \
|
||||
libssl-dev \
|
||||
libmysqlclient-dev \
|
||||
imagemagick \
|
||||
ruby-rmagick \
|
||||
@@ -19,7 +16,7 @@ sudo apt-get install -y gdal-bin \
|
||||
|
||||
# Installing system tests dependencies
|
||||
sudo apt-get install -y fonts-liberation \
|
||||
libappindicator3-1 \
|
||||
libayatana-appindicator \
|
||||
libasound2 \
|
||||
libatk-bridge2.0-0 \
|
||||
libatspi2.0-0 \
|
||||
|
||||
367
lib/exporter.rb
367
lib/exporter.rb
@@ -1,367 +0,0 @@
|
||||
class Exporter
|
||||
|
||||
def self.ulimit
|
||||
# use ulimit to restrict to 7200 CPU seconds and 5gb virtual memory, and 5gB file storage:
|
||||
#"ulimit -t 7200 && ulimit -v 5000000 && ulimit -f 5000000 && "
|
||||
"ulimit -t 14400 && ulimit -v 5000000 && ulimit -f 10000000 && nice -n 19 "
|
||||
end
|
||||
|
||||
def self.get_working_directory(path)
|
||||
"public/warps/" + path + "-working/"
|
||||
end
|
||||
|
||||
def self.warps_directory(path)
|
||||
"public/warps/" + path + "/"
|
||||
end
|
||||
|
||||
def self.delete_temp_files(path)
|
||||
system('rm -r ' + get_working_directory(path))
|
||||
system('rm ' + warps_directory(path) + '*.png')
|
||||
end
|
||||
|
||||
########################
|
||||
## Run on each image:
|
||||
|
||||
# pixels per meter = pxperm
|
||||
def self.generate_perspectival_distort(pxperm, path, nodes_array, id, image_file_name, image, height, width)
|
||||
require 'net/http'
|
||||
|
||||
# everything in -working/ can be deleted;
|
||||
# this is just so we can use the files locally outside of s3
|
||||
working_directory = get_working_directory(path)
|
||||
Dir.mkdir(working_directory) unless (File.exists?(working_directory) && File.directory?(working_directory))
|
||||
local_location = working_directory+id.to_s+'-'+image_file_name.to_s
|
||||
|
||||
directory = warps_directory(path)
|
||||
Dir.mkdir(directory) unless (File.exists?(directory) && File.directory?(directory))
|
||||
completed_local_location = directory+id.to_s+'.png'
|
||||
|
||||
# everything -masked.png can be deleted
|
||||
masked_local_location = directory+id.to_s+'-masked.png'
|
||||
# everything -mask.png can be deleted
|
||||
mask_location = directory+id.to_s+'-mask.png'
|
||||
#completed_local_location = directory+id.to_s+'.tif'
|
||||
# know everything -unwarped can be deleted
|
||||
geotiff_location = directory+id.to_s+'-geo-unwarped.tif'
|
||||
# everything -geo WITH AN ID could be deleted, but there is a feature request to preserve these
|
||||
warped_geotiff_location = directory+id.to_s+'-geo.tif'
|
||||
|
||||
northmost = nodes_array.first.lat
|
||||
southmost = nodes_array.first.lat
|
||||
westmost = nodes_array.first.lon
|
||||
eastmost = nodes_array.first.lon
|
||||
|
||||
nodes_array.each do |node|
|
||||
northmost = node.lat if node.lat > northmost
|
||||
southmost = node.lat if node.lat < southmost
|
||||
westmost = node.lon if node.lon < westmost
|
||||
eastmost = node.lon if node.lon > eastmost
|
||||
end
|
||||
|
||||
# puts northmost.to_s+','+southmost.to_s+','+westmost.to_s+','+eastmost.to_s
|
||||
|
||||
scale = 20037508.34
|
||||
y1 = pxperm*Cartagen.spherical_mercator_lat_to_y(northmost,scale)
|
||||
x1 = pxperm*Cartagen.spherical_mercator_lon_to_x(westmost,scale)
|
||||
y2 = pxperm*Cartagen.spherical_mercator_lat_to_y(southmost,scale)
|
||||
x2 = pxperm*Cartagen.spherical_mercator_lon_to_x(eastmost,scale)
|
||||
# puts x1.to_s+','+y1.to_s+','+x2.to_s+','+y2.to_s
|
||||
|
||||
# should determine if it's stored in s3 or locally:
|
||||
if (image.url[0..3] == 'http')
|
||||
Net::HTTP.start('s3.amazonaws.com') { |http|
|
||||
#Net::HTTP.start('localhost') { |http|
|
||||
puts (image.url)
|
||||
resp = http.get(image.url)
|
||||
open(local_location, "wb") { |file|
|
||||
file.write(resp.body)
|
||||
}
|
||||
}
|
||||
else
|
||||
require "fileutils"
|
||||
FileUtils.cp(Rails.root.to_s+'/public'+image.to_s,local_location)
|
||||
end
|
||||
|
||||
points = ""
|
||||
maskpoints = ""
|
||||
coordinates = ""
|
||||
first = true
|
||||
|
||||
#EXIF orientation values:
|
||||
#Value 0th Row 0th Column
|
||||
#1 top left side
|
||||
#2 top right side
|
||||
#3 bottom right side
|
||||
#4 bottom left side
|
||||
#5 left side top
|
||||
#6 right side top
|
||||
#7 right side bottom
|
||||
#8 left side bottom
|
||||
|
||||
rotation = (`identify -format %[exif:Orientation] #{local_location}`).to_i
|
||||
#stdin, stdout, stderr = Open3.popen3('identify -format %[exif:Orientation] #{local_location}')
|
||||
#rotation = stdout.readlines.first.to_s.to_i
|
||||
#puts stderr.readlines
|
||||
|
||||
if rotation == 6
|
||||
puts 'rotated CCW'
|
||||
source_corners = source_corners = [[0,height],[0,0],[width,0],[width,height]]
|
||||
elsif rotation == 8
|
||||
puts 'rotated CW'
|
||||
source_corners = [[width,0],[width,height],[0,height],[0,0]]
|
||||
elsif rotation == 3
|
||||
puts 'rotated 180 deg'
|
||||
source_corners = [[width,height],[0,height],[0,0],[width,0]]
|
||||
else
|
||||
source_corners = [[0,0],[width,0],[width,height],[0,height]]
|
||||
end
|
||||
|
||||
maxdimension = 0
|
||||
|
||||
nodes_array.each do |node|
|
||||
corner = source_corners.shift
|
||||
nx1 = corner[0]
|
||||
ny1 = corner[1]
|
||||
nx2 = -x1+(pxperm*Cartagen.spherical_mercator_lon_to_x(node.lon,scale))
|
||||
ny2 = y1-(pxperm*Cartagen.spherical_mercator_lat_to_y(node.lat,scale))
|
||||
|
||||
points = points + ' ' unless first
|
||||
maskpoints = maskpoints + ' ' unless first
|
||||
points = points + nx1.to_s + ',' + ny1.to_s + ' ' + nx2.to_i.to_s + ',' + ny2.to_i.to_s
|
||||
maskpoints = maskpoints + nx2.to_i.to_s + ',' + ny2.to_i.to_s
|
||||
first = false
|
||||
# we need to find an origin; find northwestern-most point
|
||||
coordinates = coordinates+' -gcp '+nx2.to_s+', '+ny2.to_s+', '+node.lon.to_s + ', ' + node.lat.to_s
|
||||
|
||||
# identify largest dimension to set canvas size for ImageMagick:
|
||||
maxdimension = nx1.to_i if maxdimension < nx1.to_i
|
||||
maxdimension = ny1.to_i if maxdimension < ny1.to_i
|
||||
maxdimension = nx2.to_i if maxdimension < nx2.to_i
|
||||
maxdimension = ny2.to_i if maxdimension < ny2.to_i
|
||||
end
|
||||
|
||||
# close mask polygon:
|
||||
maskpoints = maskpoints + ' '
|
||||
nx2 = -x1+(pxperm*Cartagen.spherical_mercator_lon_to_x(nodes_array.first.lon,scale))
|
||||
ny2 = y1-(pxperm*Cartagen.spherical_mercator_lat_to_y(nodes_array.first.lat,scale))
|
||||
maskpoints = maskpoints + nx2.to_i.to_s + ',' + ny2.to_i.to_s
|
||||
|
||||
height = (y1-y2).to_i.to_s
|
||||
width = (-x1+x2).to_i.to_s
|
||||
|
||||
# http://www.imagemagick.org/discourse-server/viewtopic.php?f=1&t=11319
|
||||
# http://www.imagemagick.org/discourse-server/viewtopic.php?f=3&t=8764
|
||||
# read about equalization
|
||||
# -equalize
|
||||
# -contrast-stretch 0
|
||||
|
||||
imageMagick = "convert "
|
||||
imageMagick += "-contrast-stretch 0 "
|
||||
imageMagick += local_location+" "
|
||||
imageMagick += "-crop "+maxdimension.to_i.to_s+"x"+maxdimension.to_i.to_s+"+0+0! "
|
||||
imageMagick += "-flatten "
|
||||
imageMagick += "-distort Perspective '"+points+"' "
|
||||
imageMagick += "-flatten "
|
||||
if width > height
|
||||
imageMagick += "-crop "+width+"x"+width+"+0+0\! "
|
||||
else
|
||||
imageMagick += "-crop "+height+"x"+height+"+0+0\! "
|
||||
end
|
||||
imageMagick += "+repage "
|
||||
imageMagick += completed_local_location
|
||||
puts imageMagick
|
||||
system(self.ulimit+imageMagick)
|
||||
|
||||
# create a mask (later we can blur edges here)
|
||||
imageMagick2 = 'convert +antialias '
|
||||
if width > height
|
||||
imageMagick2 += "-size "+width+"x"+width+" "
|
||||
else
|
||||
imageMagick2 += "-size "+height+"x"+height+" "
|
||||
end
|
||||
# attempt at blurred edges in masking, but I've given up, as gdal_merge doesn't seem to respect variable-opacity alpha channels
|
||||
imageMagick2 += ' xc:none -draw "fill black stroke red stroke-width 30 polyline '
|
||||
imageMagick2 += maskpoints + '" '
|
||||
imageMagick2 += ' -alpha set -channel A -transparent red -blur 0x8 -channel R -evaluate set 0 +channel '+mask_location
|
||||
#imageMagick2 += ' xc:none -draw "fill black stroke none polyline '
|
||||
#imageMagick2 += maskpoints + '" '
|
||||
#imageMagick2 += ' '+mask_location
|
||||
puts imageMagick2
|
||||
system(self.ulimit+imageMagick2)
|
||||
|
||||
imageMagick3 = 'composite '+mask_location+' '+completed_local_location+' -compose DstIn -alpha Set '+masked_local_location
|
||||
puts imageMagick3
|
||||
system(self.ulimit+imageMagick3)
|
||||
|
||||
gdal_translate = "gdal_translate -of GTiff -a_srs EPSG:4326 "+coordinates+' -co "TILED=NO" '+masked_local_location+' '+geotiff_location
|
||||
puts gdal_translate
|
||||
system(self.ulimit+gdal_translate)
|
||||
|
||||
#gdalwarp = 'gdalwarp -srcnodata "255" -dstnodata 0 -cblend 30 -of GTiff -t_srs EPSG:4326 '+geotiff_location+' '+warped_geotiff_location
|
||||
gdalwarp = 'gdalwarp -of GTiff -t_srs EPSG:4326 '+geotiff_location+' '+warped_geotiff_location
|
||||
puts gdalwarp
|
||||
system(self.ulimit+gdalwarp)
|
||||
|
||||
# deletions could happen here; do it in distinct method so we can run it independently
|
||||
delete_temp_files(path)
|
||||
|
||||
[x1,y1]
|
||||
end
|
||||
|
||||
########################
|
||||
## Run on maps:
|
||||
|
||||
# distort all warpables, returns upper left corner coords in x,y
|
||||
def self.distort_warpables(scale, warpables, export, slug)
|
||||
|
||||
puts '> generating geotiffs of each warpable in GDAL'
|
||||
lowest_x=0
|
||||
lowest_y=0
|
||||
warpable_coords = []
|
||||
current = 0
|
||||
warpables.each do |warpable|
|
||||
current += 1
|
||||
|
||||
## TODO: refactor to generate static status file:
|
||||
export.status = 'warping '+current.to_s+' of '+warpables.length.to_s
|
||||
puts 'warping '+current.to_s+' of '+warpables.length.to_s
|
||||
export.save
|
||||
##
|
||||
|
||||
my_warpable_coords = warpable.generate_perspectival_distort(scale,slug)
|
||||
puts '- '+my_warpable_coords.to_s
|
||||
warpable_coords << my_warpable_coords
|
||||
lowest_x = my_warpable_coords.first if (my_warpable_coords.first < lowest_x || lowest_x.zero? )
|
||||
lowest_y = my_warpable_coords.last if (my_warpable_coords.last < lowest_y || lowest_y.zero? )
|
||||
end
|
||||
[lowest_x,lowest_y,warpable_coords]
|
||||
end
|
||||
|
||||
# generate a tiff from all warpable images in this set
|
||||
def self.generate_composite_tiff(coords, origin, placed_warpables, slug, ordered)
|
||||
directory = "public/warps/"+slug+"/"
|
||||
composite_location = directory+slug+'-geo.tif'
|
||||
geotiffs = ''
|
||||
minlat = nil
|
||||
minlon = nil
|
||||
maxlat = nil
|
||||
maxlon = nil
|
||||
placed_warpables.each do |warpable|
|
||||
warpable.nodes_array.each do |n|
|
||||
minlat = n.lat if minlat == nil || n.lat < minlat
|
||||
minlon = n.lon if minlon == nil || n.lon < minlon
|
||||
maxlat = n.lat if maxlat == nil || n.lat > maxlat
|
||||
maxlon = n.lon if maxlon == nil || n.lon > maxlon
|
||||
end
|
||||
end
|
||||
if ordered != true
|
||||
# sort by area; this would be overridden by a provided order
|
||||
warpables = placed_warpables.sort{|a,b|b.poly_area <=> a.poly_area}
|
||||
end
|
||||
warpables.each do |warpable|
|
||||
geotiffs += ' '+directory+warpable.id.to_s+'-geo.tif'
|
||||
end
|
||||
gdalwarp = "gdalwarp -s_srs EPSG:3857 -te #{minlon} #{minlat} #{maxlon} #{maxlat} #{geotiffs} #{directory}#{slug}-geo.tif"
|
||||
puts gdalwarp
|
||||
system(self.ulimit+gdalwarp)
|
||||
composite_location
|
||||
end
|
||||
|
||||
# generates a tileset at root/public/tms/<slug>/
|
||||
# root is something like https://mapknitter.org
|
||||
def self.generate_tiles(key, slug, root)
|
||||
key = "AIzaSyAOLUQngEmJv0_zcG1xkGq-CXIPpLQY8iQ" if key.blank?
|
||||
gdal2tiles = 'gdal2tiles.py -k --s_srs EPSG:4326 -t "'+slug+'" -g "'+key+'" '+root+'/public/warps/'+slug+'/'+slug+'-geo.tif '+root+'/public/tms/'+slug+"/"
|
||||
puts gdal2tiles
|
||||
system(self.ulimit+gdal2tiles)
|
||||
end
|
||||
|
||||
# zips up tiles at root/public/tms/<slug>.zip;
|
||||
def self.zip_tiles(slug)
|
||||
rmzip = 'cd public/tms/ && rm '+slug+'.zip && cd ../../'
|
||||
system(rmzip)
|
||||
zip = 'cd public/tms/ && ' + self.ulimit + 'zip -rq '+slug+'.zip '+slug+'/ && cd ../../'
|
||||
system(zip)
|
||||
end
|
||||
|
||||
# generates a tileset at root/public/tms/<slug>/
|
||||
def self.generate_jpg(slug, root)
|
||||
imageMagick = 'convert -background white -flatten '+root+'/public/warps/'+slug+'/'+slug+'-geo.tif '+root+'/public/warps/'+slug+'/'+slug+'.jpg'
|
||||
system(self.ulimit+imageMagick)
|
||||
end
|
||||
|
||||
# runs the above map functions while maintaining a record of state in an Export model;
|
||||
# we'll be replacing the export model state with a flat status file
|
||||
def self.run_export(user,resolution,export,id,slug,root,average_scale,placed_warpables,key)
|
||||
begin
|
||||
export.user_id = user.id if user
|
||||
export.status = 'starting'
|
||||
export.tms = false
|
||||
export.geotiff = false
|
||||
export.zip = false
|
||||
export.jpg = false
|
||||
export.bands_string = 'default bands_string'
|
||||
export.save
|
||||
|
||||
directory = "#{root}/public/warps/"+slug+"/"
|
||||
stdin, stdout, stderr = Open3.popen3('rm -r '+directory.to_s)
|
||||
puts stdout.readlines
|
||||
puts stderr.readlines
|
||||
stdin, stdout, stderr = Open3.popen3("rm -r #{root}/public/tms/#{slug}")
|
||||
puts stdout.readlines
|
||||
puts stderr.readlines
|
||||
|
||||
puts '> averaging scales; resolution: ' + resolution.to_s
|
||||
pxperm = 100/(resolution).to_f || average_scale # pixels per meter
|
||||
puts '> scale: ' + pxperm.to_s + 'pxperm'
|
||||
|
||||
puts '> distorting warpables'
|
||||
|
||||
origin = self.distort_warpables(pxperm, placed_warpables, export, slug)
|
||||
warpable_coords = origin.pop
|
||||
|
||||
export.status = 'compositing'
|
||||
export.save
|
||||
|
||||
puts '> generating composite tiff'
|
||||
composite_location = self.generate_composite_tiff(warpable_coords,origin,placed_warpables,slug,false) # no ordering yet
|
||||
|
||||
info = (`identify -quiet -format '%b,%w,%h' #{composite_location}`).split(',')
|
||||
puts info
|
||||
|
||||
if info[0] != ''
|
||||
export.geotiff = true
|
||||
export.size = info[0]
|
||||
export.width = info[1]
|
||||
export.height = info[2]
|
||||
export.cm_per_pixel = 100.0000/pxperm
|
||||
export.status = 'tiling'
|
||||
export.save
|
||||
end
|
||||
|
||||
puts '> generating tiles'
|
||||
export.tms = true if self.generate_tiles(key, slug, root)
|
||||
export.status = 'zipping tiles'
|
||||
export.save
|
||||
|
||||
puts '> zipping tiles'
|
||||
export.zip = true if self.zip_tiles(slug)
|
||||
export.status = 'creating jpg'
|
||||
export.save
|
||||
|
||||
puts '> generating jpg'
|
||||
export.jpg = true if self.generate_jpg(slug, root)
|
||||
export.status = 'complete'
|
||||
export.save
|
||||
|
||||
rescue SystemCallError => e
|
||||
puts "ERROR"
|
||||
puts e.inspect
|
||||
puts "END ERROR"
|
||||
export.status = 'failed'
|
||||
export.save
|
||||
end
|
||||
return export.status
|
||||
end
|
||||
|
||||
end
|
||||
@@ -22,87 +22,10 @@ class ExportControllerTest < ActionController::TestCase
|
||||
assert_equal url, Export.last.export_url
|
||||
end
|
||||
|
||||
test 'should create jpg after export' do
|
||||
map = maps(:cubbon)
|
||||
system('mkdir -p public/warps/cubbon-park')
|
||||
system('cp test/fixtures/demo.png public/warps/cubbon-park/cubbon-park.jpg')
|
||||
|
||||
get :jpg, params: { id: map.slug}
|
||||
assert_response :success
|
||||
assert_includes 'image/jpeg', response.content_type
|
||||
end
|
||||
|
||||
test 'should create geotiff after export' do
|
||||
map = maps(:cubbon)
|
||||
system('mkdir -p public/warps/cubbon-park')
|
||||
system('cp test/fixtures/demo.png public/warps/cubbon-park/cubbon-park-geo.tif')
|
||||
|
||||
get :geotiff, params: { id: map.slug}
|
||||
assert_response :success
|
||||
assert_includes 'image/tiff', response.content_type
|
||||
end
|
||||
|
||||
test 'should not cancel if not logged in' do
|
||||
get :cancel, params: { id: @map.id}
|
||||
assert_response :success
|
||||
assert_equal 'You must be logged in to export, unless the map is anonymous.', @response.body
|
||||
assert assigns[:map]
|
||||
assert_equal 'text/plain', @response.content_type
|
||||
assert flash.empty?
|
||||
end
|
||||
|
||||
test 'should cancel export' do
|
||||
session[:user_id] = 1
|
||||
get :cancel, params: { id: @map.id}
|
||||
assert_response :success
|
||||
assert_equal 'cancelled', @response.body
|
||||
assert assigns[:map]
|
||||
end
|
||||
|
||||
test 'should redirect after cancelling' do
|
||||
session[:user_id] = 1
|
||||
get :cancel, params: { id: @map.id, exports: 'cess'}
|
||||
assert_response :redirect
|
||||
assert flash.present?
|
||||
assert_redirected_to '/exports'
|
||||
end
|
||||
|
||||
test 'should display export progress' do
|
||||
get :progress, params: { id: @map.id}
|
||||
assert_response :success
|
||||
assert_equal 'export not running', @response.body
|
||||
assert_equal 'text/plain', @response.content_type
|
||||
end
|
||||
|
||||
test 'should display progress with no export' do
|
||||
get :progress, params: { id: 4}
|
||||
assert_response :success
|
||||
assert_equal 'export has not been run', @response.body
|
||||
end
|
||||
|
||||
test 'should display progress completed' do
|
||||
get :progress, params: { id: 2}
|
||||
assert_response :success
|
||||
assert_equal 'complete', @response.body
|
||||
end
|
||||
|
||||
test 'should display progress failed' do
|
||||
get :progress, params: { id: 3}
|
||||
assert_response :success
|
||||
assert_equal 'export failed', @response.body
|
||||
end
|
||||
|
||||
# does not test the exporter client
|
||||
test 'should display export status' do
|
||||
session[:user_id] = 1
|
||||
get :status, params: { id: @map.id}
|
||||
assert_response :success
|
||||
end
|
||||
|
||||
test 'should display error if no export' do
|
||||
session[:user_id] = 1
|
||||
get :status, params: { id: 4}
|
||||
assert_response :success
|
||||
# assert_equal { status: 'export has not been run' }.to_json, @response.body
|
||||
end
|
||||
end
|
||||
|
||||
@@ -1,21 +0,0 @@
|
||||
require 'test_helper'
|
||||
|
||||
class ExportTest < ActiveSupport::TestCase
|
||||
test 'should count methods' do
|
||||
average = Export.all.map(&:cm_per_pixel).sum/Export.count
|
||||
assert_not_nil Export.average_cm_per_pixel
|
||||
assert_equal average, Export.average_cm_per_pixel
|
||||
assert_not_nil Export.histogram_cm_per_pixel
|
||||
assert_not_nil Export.histogram_cm_per_pixel_in_tens
|
||||
assert_not_nil Export.export_count
|
||||
assert_not_nil Export.exporting
|
||||
|
||||
Export.delete_all
|
||||
assert_empty Export.histogram_cm_per_pixel
|
||||
assert_equal 0, Export.average_cm_per_pixel
|
||||
end
|
||||
|
||||
test 'should export running' do
|
||||
assert !Export.last.running?
|
||||
end
|
||||
end
|
||||
@@ -1,57 +0,0 @@
|
||||
require 'test_helper'
|
||||
|
||||
class ExporterTest < ActiveSupport::TestCase
|
||||
test 'should export warpable using isolated exporter lib' do
|
||||
|
||||
# make a sample image
|
||||
system('mkdir -p public/system/images/1/original')
|
||||
system('cp test/fixtures/demo.png public/system/images/1/original/')
|
||||
system('mkdir -p public/warps/saugus-landfill-incinerator')
|
||||
system('mkdir -p public/tms/saugus-landfill-incinerator')
|
||||
system('touch public/warps/saugus-landfill-incinerator/folder')
|
||||
assert File.exist?('public/warps/saugus-landfill-incinerator/folder')
|
||||
|
||||
scale = 2
|
||||
|
||||
w = warpables(:one)
|
||||
coords = Exporter.generate_perspectival_distort(scale, w.map.slug, w.nodes_array, w.id, w.image_file_name, w.image, w.height, w.width)
|
||||
assert coords
|
||||
assert Exporter.get_working_directory(w.map.slug)
|
||||
assert Exporter.warps_directory(w.map.slug)
|
||||
|
||||
map = maps(:saugus)
|
||||
|
||||
# get rid of existing geotiff
|
||||
system('rm -r public/warps/saugus-landfill-incinerator/1-geo.tif')
|
||||
# make a sample image
|
||||
system('mkdir -p public/system/images/2/original/')
|
||||
system('cp test/fixtures/demo.png public/system/images/2/original/test.png')
|
||||
origin = Exporter.distort_warpables(scale, map.warpables, map.export, map.slug)
|
||||
lowest_x, lowest_y, warpable_coords = origin
|
||||
assert origin
|
||||
ordered = false
|
||||
|
||||
system('mkdir -p public/warps/saugus-landfill-incinerator')
|
||||
system('mkdir -p public/tms/saugus-landfill-incinerator')
|
||||
# these params could be compressed - warpable coords is part of origin; are coords and origin required?
|
||||
assert Exporter.generate_composite_tiff(warpable_coords, origin, map.placed_warpables, map.slug, ordered)
|
||||
assert Exporter.generate_tiles('', map.slug, Rails.root.to_s)
|
||||
assert Exporter.zip_tiles(map.slug)
|
||||
assert Exporter.generate_jpg(map.slug, Rails.root.to_s)
|
||||
resolution = 20
|
||||
assert Exporter.run_export(User.last, resolution, map.export, map.id, map.slug, Rails.root.to_s, map.average_scale, map.placed_warpables, '')
|
||||
|
||||
# test deletion of the files; they were already deleted in run_export, so let's make sample ones:
|
||||
# make a sample image
|
||||
system('mkdir -p public/system/images/2/original/')
|
||||
system('touch public/system/images/2/original/test.png')
|
||||
system('mkdir -p public/warps/saugus-landfill-incinerator')
|
||||
system('mkdir -p public/tms/saugus-landfill-incinerator')
|
||||
system('touch public/warps/saugus-landfill-incinerator/folder')
|
||||
assert File.exist?('public/warps/saugus-landfill-incinerator/folder')
|
||||
system('mkdir -p public/warps/saugus-landfill-incinerator-working')
|
||||
system('touch public/warps/saugus-landfill-incinerator/test.png')
|
||||
assert Exporter.delete_temp_files(w.map.slug)
|
||||
end
|
||||
end
|
||||
|
||||
@@ -49,42 +49,6 @@ class MapTest < ActiveSupport::TestCase
|
||||
assert_not_nil map.latest_export
|
||||
assert_not_nil map.nodes
|
||||
assert_not_nil map.average_cm_per_pixel
|
||||
|
||||
# use a map fixture with no warpables
|
||||
village = maps(:village)
|
||||
assert_equal 0, village.average_cm_per_pixel
|
||||
|
||||
resolution = 20
|
||||
assert_not_nil map.run_export(users(:quentin), resolution) #map.average_cm_per_pixel)
|
||||
|
||||
# main issue will be that it creates and continuously updates an Export model.
|
||||
# we could shift this to a polling model, either on the client side (eliminating the Export model)
|
||||
# ... or some other way to make it possible to do many of these tasks without needing ActiveRecord
|
||||
|
||||
# let's start at the bottom and factor this all out working upwards
|
||||
|
||||
# refactor so that we pass in as much in parameters as possible, reducing in-model cross-references
|
||||
|
||||
# creates an Export and sets initial values
|
||||
# depends on: map.average_scale
|
||||
# runs self.distort_warpables(pxperm)
|
||||
# runs self.generate_composite_tiff(warpable_coords,origin)
|
||||
# runs `identify` and assigns some values (height, width) to Export
|
||||
# runs export.tms = true if self.generate_tiles
|
||||
# runs export.zip = true if self.zip_tiles
|
||||
# runs export.jpg = true if self.generate_jpg
|
||||
# map.distort_warpables(scale)
|
||||
# collects self.placed_warpables
|
||||
# runs on each one: warpable.generate_perspectival_distort(scale,self.slug)
|
||||
# map.generate_composite_tiff(coords,origin)
|
||||
# collects self.placed_warpables
|
||||
# runs gdal_warp on the output of each, flattening onto a single geotiff
|
||||
# map.generate_tiles
|
||||
# runs on composite tiff output: gdal2tiles = 'gdal2tiles.py -k -t "'+self.slug+'" -g "'+google_api_key+'" '+Rails.root.to_s+'/public/warps/'+self.slug+'/'+self.slug+'-geo.tif '+Rails.root.to_s+'/public/tms/'+self.slug+"/"
|
||||
# map.zip_tiles
|
||||
# map.generate_jpg
|
||||
# runs convert on composite tiff
|
||||
|
||||
end
|
||||
|
||||
test 'should have histograms' do
|
||||
@@ -94,7 +58,6 @@ class MapTest < ActiveSupport::TestCase
|
||||
assert_not_nil hist
|
||||
assert_not_nil map.grouped_images_histogram(3)
|
||||
assert_equal hist.count/3, map.grouped_images_histogram(3).count
|
||||
|
||||
end
|
||||
|
||||
test 'should have nearby maps' do
|
||||
|
||||
@@ -53,33 +53,4 @@ class WarpableTest < ActiveSupport::TestCase
|
||||
Warpable.delete_all
|
||||
assert_empty Warpable.histogram_cm_per_pixel
|
||||
end
|
||||
|
||||
test 'should try export warpables' do
|
||||
# make a sample image
|
||||
system('mkdir -p public/warps/saugus-landfill-incinerator-working')
|
||||
system('mkdir -p public/system/images/1/original')
|
||||
system('cp test/fixtures/demo.png public/system/images/1/original/')
|
||||
system('mkdir -p public/warps/saugus-landfill-incinerator')
|
||||
system('touch public/warps/saugus-landfill-incinerator/folder')
|
||||
system('mkdir -p public/system/images/2/original/')
|
||||
system('cp test/fixtures/demo.png public/system/images/2/original/test.png')
|
||||
system('cp test/fixtures/demo.png public/warps/saugus-landfill-incinerator/1-geo.tif')
|
||||
assert File.exist?('public/warps/saugus-landfill-incinerator/folder')
|
||||
|
||||
origin = Exporter.distort_warpables(2,
|
||||
@map.warpables,
|
||||
@map.export,
|
||||
@map.slug)
|
||||
|
||||
assert_not_nil @warp.save_dimensions
|
||||
assert_not_nil @warp.user_id
|
||||
|
||||
Exporter.generate_composite_tiff(nil,
|
||||
origin,
|
||||
@map.placed_warpables,
|
||||
@map.slug,
|
||||
false)
|
||||
|
||||
assert File.exist?('public/warps/saugus-landfill-incinerator/1-geo.tif')
|
||||
end
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user