eessppeelllloo/mapknitter
1
0
Fork 0
mirror of https://github.com/publiclab/mapknitter.git synced 2025-12-15 04:40:00 +01:00
Code Issues Packages Projects Releases Wiki Activity
Files
c76748ec33056db9baccfca0717e360e51a2bc74
mapknitter/lib/exporter.rb
jywarren c76748ec33 exporter hotfix
2019-08-14 19:02:54 -04:00

368 lines
13 KiB
Ruby
Raw Blame History

class Exporter
def self.ulimit
# use ulimit to restrict to 7200 CPU seconds and 5gb virtual memory, and 5gB file storage:
#"ulimit -t 7200 && ulimit -v 5000000 && ulimit -f 5000000 && "
"ulimit -t 14400 && ulimit -v 5000000 && ulimit -f 10000000 && nice -n 19 "
end
def self.get_working_directory(path)
"public/warps/" + path + "-working/"
end
def self.warps_directory(path)
"public/warps/" + path + "/"
end
def self.delete_temp_files(path)
system('rm -r ' + get_working_directory(path))
system('rm ' + warps_directory(path) + '*.png')
end
########################
## Run on each image:
# pixels per meter = pxperm
def self.generate_perspectival_distort(pxperm, path, nodes_array, id, image_file_name, image, height, width)
require 'net/http'
# everything in -working/ can be deleted;
# this is just so we can use the files locally outside of s3
working_directory = get_working_directory(path)
Dir.mkdir(working_directory) unless (File.exists?(working_directory) && File.directory?(working_directory))
local_location = working_directory+id.to_s+'-'+image_file_name.to_s
directory = warps_directory(path)
Dir.mkdir(directory) unless (File.exists?(directory) && File.directory?(directory))
completed_local_location = directory+id.to_s+'.png'
# everything -masked.png can be deleted
masked_local_location = directory+id.to_s+'-masked.png'
# everything -mask.png can be deleted
mask_location = directory+id.to_s+'-mask.png'
#completed_local_location = directory+id.to_s+'.tif'
# know everything -unwarped can be deleted
geotiff_location = directory+id.to_s+'-geo-unwarped.tif'
# everything -geo WITH AN ID could be deleted, but there is a feature request to preserve these
warped_geotiff_location = directory+id.to_s+'-geo.tif'
northmost = nodes_array.first.lat
southmost = nodes_array.first.lat
westmost = nodes_array.first.lon
eastmost = nodes_array.first.lon
nodes_array.each do |node|
northmost = node.lat if node.lat > northmost
southmost = node.lat if node.lat < southmost
westmost = node.lon if node.lon < westmost
eastmost = node.lon if node.lon > eastmost
end
# puts northmost.to_s+','+southmost.to_s+','+westmost.to_s+','+eastmost.to_s
scale = 20037508.34
y1 = pxperm*Cartagen.spherical_mercator_lat_to_y(northmost,scale)
x1 = pxperm*Cartagen.spherical_mercator_lon_to_x(westmost,scale)
y2 = pxperm*Cartagen.spherical_mercator_lat_to_y(southmost,scale)
x2 = pxperm*Cartagen.spherical_mercator_lon_to_x(eastmost,scale)
# puts x1.to_s+','+y1.to_s+','+x2.to_s+','+y2.to_s
# should determine if it's stored in s3 or locally:
if (image.url[0..3] == 'http')
Net::HTTP.start('s3.amazonaws.com') { |http|
#Net::HTTP.start('localhost') { |http|
puts (image.url)
resp = http.get(image.url)
open(local_location, "wb") { |file|
file.write(resp.body)
}
}
else
require "fileutils"
FileUtils.cp(Rails.root.to_s+'/public'+image.to_s,local_location)
end
points = ""
maskpoints = ""
coordinates = ""
first = true
#EXIF orientation values:
#Value 0th Row 0th Column
#1 top left side
#2 top right side
#3 bottom right side
#4 bottom left side
#5 left side top
#6 right side top
#7 right side bottom
#8 left side bottom
rotation = (`identify -format %[exif:Orientation] #{local_location}`).to_i
#stdin, stdout, stderr = Open3.popen3('identify -format %[exif:Orientation] #{local_location}')
#rotation = stdout.readlines.first.to_s.to_i
#puts stderr.readlines
if rotation == 6
puts 'rotated CCW'
source_corners = source_corners = [[0,height],[0,0],[width,0],[width,height]]
elsif rotation == 8
puts 'rotated CW'
source_corners = [[width,0],[width,height],[0,height],[0,0]]
elsif rotation == 3
puts 'rotated 180 deg'
source_corners = [[width,height],[0,height],[0,0],[width,0]]
else
source_corners = [[0,0],[width,0],[width,height],[0,height]]
end
maxdimension = 0
nodes_array.each do |node|
corner = source_corners.shift
nx1 = corner[0]
ny1 = corner[1]
nx2 = -x1+(pxperm*Cartagen.spherical_mercator_lon_to_x(node.lon,scale))
ny2 = y1-(pxperm*Cartagen.spherical_mercator_lat_to_y(node.lat,scale))
points = points + ' ' unless first
maskpoints = maskpoints + ' ' unless first
points = points + nx1.to_s + ',' + ny1.to_s + ' ' + nx2.to_i.to_s + ',' + ny2.to_i.to_s
maskpoints = maskpoints + nx2.to_i.to_s + ',' + ny2.to_i.to_s
first = false
# we need to find an origin; find northwestern-most point
coordinates = coordinates+' -gcp '+nx2.to_s+', '+ny2.to_s+', '+node.lon.to_s + ', ' + node.lat.to_s
# identify largest dimension to set canvas size for ImageMagick:
maxdimension = nx1.to_i if maxdimension < nx1.to_i
maxdimension = ny1.to_i if maxdimension < ny1.to_i
maxdimension = nx2.to_i if maxdimension < nx2.to_i
maxdimension = ny2.to_i if maxdimension < ny2.to_i
end
# close mask polygon:
maskpoints = maskpoints + ' '
nx2 = -x1+(pxperm*Cartagen.spherical_mercator_lon_to_x(nodes_array.first.lon,scale))
ny2 = y1-(pxperm*Cartagen.spherical_mercator_lat_to_y(nodes_array.first.lat,scale))
maskpoints = maskpoints + nx2.to_i.to_s + ',' + ny2.to_i.to_s
height = (y1-y2).to_i.to_s
width = (-x1+x2).to_i.to_s
# http://www.imagemagick.org/discourse-server/viewtopic.php?f=1&t=11319
# http://www.imagemagick.org/discourse-server/viewtopic.php?f=3&t=8764
# read about equalization
# -equalize
# -contrast-stretch 0
imageMagick = "convert "
imageMagick += "-contrast-stretch 0 "
imageMagick += local_location+" "
imageMagick += "-crop "+maxdimension.to_i.to_s+"x"+maxdimension.to_i.to_s+"+0+0! "
imageMagick += "-flatten "
imageMagick += "-distort Perspective '"+points+"' "
imageMagick += "-flatten "
if width > height
imageMagick += "-crop "+width+"x"+width+"+0+0\! "
else
imageMagick += "-crop "+height+"x"+height+"+0+0\! "
end
imageMagick += "+repage "
imageMagick += completed_local_location
puts imageMagick
system(self.ulimit+imageMagick)
# create a mask (later we can blur edges here)
imageMagick2 = 'convert +antialias '
if width > height
imageMagick2 += "-size "+width+"x"+width+" "
else
imageMagick2 += "-size "+height+"x"+height+" "
end
# attempt at blurred edges in masking, but I've given up, as gdal_merge doesn't seem to respect variable-opacity alpha channels
imageMagick2 += ' xc:none -draw "fill black stroke red stroke-width 30 polyline '
imageMagick2 += maskpoints + '" '
imageMagick2 += ' -alpha set -channel A -transparent red -blur 0x8 -channel R -evaluate set 0 +channel '+mask_location
#imageMagick2 += ' xc:none -draw "fill black stroke none polyline '
#imageMagick2 += maskpoints + '" '
#imageMagick2 += ' '+mask_location
puts imageMagick2
system(self.ulimit+imageMagick2)
imageMagick3 = 'composite '+mask_location+' '+completed_local_location+' -compose DstIn -alpha Set '+masked_local_location
puts imageMagick3
system(self.ulimit+imageMagick3)
gdal_translate = "gdal_translate -of GTiff -a_srs EPSG:4326 "+coordinates+' -co "TILED=NO" '+masked_local_location+' '+geotiff_location
puts gdal_translate
system(self.ulimit+gdal_translate)
#gdalwarp = 'gdalwarp -srcnodata "255" -dstnodata 0 -cblend 30 -of GTiff -t_srs EPSG:4326 '+geotiff_location+' '+warped_geotiff_location
gdalwarp = 'gdalwarp -of GTiff -t_srs EPSG:4326 '+geotiff_location+' '+warped_geotiff_location
puts gdalwarp
system(self.ulimit+gdalwarp)
# deletions could happen here; do it in distinct method so we can run it independently
delete_temp_files(path)
[x1,y1]
end
########################
## Run on maps:
# distort all warpables, returns upper left corner coords in x,y
def self.distort_warpables(scale, warpables, export, slug)
puts '> generating geotiffs of each warpable in GDAL'
lowest_x=0
lowest_y=0
warpable_coords = []
current = 0
warpables.each do |warpable|
current += 1
## TODO: refactor to generate static status file:
export.status = 'warping '+current.to_s+' of '+warpables.length.to_s
puts 'warping '+current.to_s+' of '+warpables.length.to_s
export.save
##
my_warpable_coords = warpable.generate_perspectival_distort(scale,slug)
puts '- '+my_warpable_coords.to_s
warpable_coords << my_warpable_coords
lowest_x = my_warpable_coords.first if (my_warpable_coords.first < lowest_x || lowest_x.zero? )
lowest_y = my_warpable_coords.last if (my_warpable_coords.last < lowest_y || lowest_y.zero? )
end
[lowest_x,lowest_y,warpable_coords]
end
# generate a tiff from all warpable images in this set
def self.generate_composite_tiff(coords, origin, placed_warpables, slug, ordered)
directory = "public/warps/"+slug+"/"
composite_location = directory+slug+'-geo.tif'
geotiffs = ''
minlat = nil
minlon = nil
maxlat = nil
maxlon = nil
placed_warpables.each do |warpable|
warpable.nodes_array.each do |n|
minlat = n.lat if minlat == nil || n.lat < minlat
minlon = n.lon if minlon == nil || n.lon < minlon
maxlat = n.lat if maxlat == nil || n.lat > maxlat
maxlon = n.lon if maxlon == nil || n.lon > maxlon
end
end
if ordered != true
# sort by area; this would be overridden by a provided order
warpables = placed_warpables.sort{|a,b|b.poly_area <=> a.poly_area}
end
warpables.each do |warpable|
geotiffs += ' '+directory+warpable.id.to_s+'-geo.tif'
end
gdalwarp = "gdalwarp -s_srs EPSG:3857 -te #{minlon} #{minlat} #{maxlon} #{maxlat} #{geotiffs} #{directory}#{id}.tif"
puts gdalwarp
system(self.ulimit+gdalwarp)
composite_location
end
# generates a tileset at root/public/tms/<slug>/
# root is something like https://mapknitter.org
def self.generate_tiles(key, slug, root)
key = "AIzaSyAOLUQngEmJv0_zcG1xkGq-CXIPpLQY8iQ" if key.blank?
gdal2tiles = 'gdal2tiles.py -k --s_srs EPSG:4326 -t "'+slug+'" -g "'+key+'" '+root+'/public/warps/'+slug+'/'+slug+'-geo.tif '+root+'/public/tms/'+slug+"/"
puts gdal2tiles
system(self.ulimit+gdal2tiles)
end
# zips up tiles at root/public/tms/<slug>.zip;
def self.zip_tiles(slug)
rmzip = 'cd public/tms/ && rm '+slug+'.zip && cd ../../'
system(rmzip)
zip = 'cd public/tms/ && ' + self.ulimit + 'zip -rq '+slug+'.zip '+slug+'/ && cd ../../'
system(zip)
end
# generates a tileset at root/public/tms/<slug>/
def self.generate_jpg(slug, root)
imageMagick = 'convert -background white -flatten '+root+'/public/warps/'+slug+'/'+slug+'-geo.tif '+root+'/public/warps/'+slug+'/'+slug+'.jpg'
system(self.ulimit+imageMagick)
end
# runs the above map functions while maintaining a record of state in an Export model;
# we'll be replacing the export model state with a flat status file
def self.run_export(user,resolution,export,id,slug,root,average_scale,placed_warpables,key)
begin
export.user_id = user.id if user
export.status = 'starting'
export.tms = false
export.geotiff = false
export.zip = false
export.jpg = false
export.bands_string = 'default bands_string'
export.save
directory = "#{root}/public/warps/"+slug+"/"
stdin, stdout, stderr = Open3.popen3('rm -r '+directory.to_s)
puts stdout.readlines
puts stderr.readlines
stdin, stdout, stderr = Open3.popen3("rm -r #{root}/public/tms/#{slug}")
puts stdout.readlines
puts stderr.readlines
puts '> averaging scales; resolution: ' + resolution.to_s
pxperm = 100/(resolution).to_f || average_scale # pixels per meter
puts '> scale: ' + pxperm.to_s + 'pxperm'
puts '> distorting warpables'
origin = self.distort_warpables(pxperm, placed_warpables, export, slug)
warpable_coords = origin.pop
export.status = 'compositing'
export.save
puts '> generating composite tiff'
composite_location = self.generate_composite_tiff(warpable_coords,origin,placed_warpables,slug,false) # no ordering yet
info = (`identify -quiet -format '%b,%w,%h' #{composite_location}`).split(',')
puts info
if info[0] != ''
export.geotiff = true
export.size = info[0]
export.width = info[1]
export.height = info[2]
export.cm_per_pixel = 100.0000/pxperm
export.status = 'tiling'
export.save
end
puts '> generating tiles'
export.tms = true if self.generate_tiles(key, slug, root)
export.status = 'zipping tiles'
export.save
puts '> zipping tiles'
export.zip = true if self.zip_tiles(slug)
export.status = 'creating jpg'
export.save
puts '> generating jpg'
export.jpg = true if self.generate_jpg(slug, root)
export.status = 'complete'
export.save
rescue SystemCallError => e
puts "ERROR"
puts e.inspect
puts "END ERROR"
export.status = 'failed'
export.save
end
return export.status
end
end
Reference in New Issue View Git Blame Copy Permalink