284 lines
9.4 KiB
Perl
Executable File
284 lines
9.4 KiB
Perl
Executable File
#!/usr/bin/perl
|
|
# Configuration in env: OSM_CACHE_DIR, OSM_CARTO_VERSION, URL_LATEST, URL_UPDATES,
|
|
# plus PG_ENV_OSM_{DB,HOST,PORT,USER,PASSWORD} like in README for the render server itself
|
|
|
|
use strict;
|
|
use DBI;
|
|
use POSIX;
|
|
|
|
my $dbh;
|
|
my $dir = $ENV{OSM_CACHE_DIR} || '/var/lib/mod_tile/downloads';
|
|
my $method = $ENV{OSM_METHOD} || 'osm2pgsql-carto';
|
|
my $url_latest = $ENV{URL_LATEST} || 'http://download.geofabrik.de/russia-latest.osm.pbf';
|
|
my $url_updates = $ENV{URL_UPDATES} || 'http://download.geofabrik.de/russia-updates';
|
|
|
|
if (!$ENV{PG_ENV_OSM_DB})
|
|
{
|
|
fatal("OSM database not specified");
|
|
}
|
|
|
|
info("Started OSM update for database: $ENV{PG_ENV_OSM_DB}");
|
|
|
|
if ($method ne 'osm2pgsql-carto' && $method ne 'imposm3-all')
|
|
{
|
|
fatal("Incorrect OSM update mode: $method");
|
|
}
|
|
|
|
-e $dir || mkdir($dir);
|
|
chdir $dir or fatal("Failed to chdir $dir");
|
|
eval { run_update() };
|
|
if ($@)
|
|
{
|
|
fatal("$@");
|
|
}
|
|
exit(0);
|
|
|
|
sub run_update
|
|
{
|
|
my $state = parse_geofabrik_state($url_updates, $dir);
|
|
my $dbh = DBI->connect(
|
|
'dbi:Pg:dbname='.$ENV{PG_ENV_OSM_DB}.';host='.$ENV{PG_ENV_OSM_HOST}.';port='.($ENV{PG_ENV_OSM_PORT}||5432),
|
|
$ENV{PG_ENV_OSM_USER}, $ENV{PG_ENV_OSM_PASSWORD}, {AutoCommit => 0, RaiseError => 1}
|
|
);
|
|
my ($version) = eval { $dbh->selectrow_array(
|
|
'SELECT value FROM replication_state WHERE name=? FOR UPDATE', {}, 'osm_version'
|
|
) };
|
|
if (!$version)
|
|
{
|
|
if (!$ENV{OSM_INIT})
|
|
{
|
|
fatal("Current OSM version missing, run with OSM_INIT=1 environment variable to initialize");
|
|
}
|
|
$dbh->rollback; # иначе postgresql пишет "statements until the end ignored"
|
|
# создаём таблицу и оставляем её заблокированной
|
|
init_state($dbh, $state);
|
|
$dbh->do('CREATE EXTENSION IF NOT EXISTS postgis');
|
|
$dbh->do('CREATE EXTENSION IF NOT EXISTS hstore');
|
|
# качаем дамп
|
|
my ($fn) = $url_latest =~ /([^\/]+)$/so;
|
|
$fn =~ s/^([^\.]+)/$1-$state->{timestamp}/;
|
|
info("Downloading $url_latest");
|
|
system("curl -s -C - -f '$url_latest' -o $dir/$fn");
|
|
if ($? || !-e "$dir/$fn")
|
|
{
|
|
fatal("Failed to download $url_latest");
|
|
}
|
|
if ($method eq 'osm2pgsql-carto')
|
|
{
|
|
init_osm2pgsql($dbh, $state->{timestamp} . ' ' . $state->{sequenceNumber}, "$dir/$fn");
|
|
}
|
|
else
|
|
{
|
|
init_imposm3($dbh, $state->{timestamp} . ' ' . $state->{sequenceNumber}, "$dir/$fn");
|
|
}
|
|
$dbh->commit;
|
|
info("OSM database initialized with version: ".$state->{timestamp});
|
|
}
|
|
else
|
|
{
|
|
my $apply = load_geofabrik_deltas($version, $state, $url_updates, $dir);
|
|
chdir($dir);
|
|
if ($method eq 'osm2pgsql-carto')
|
|
{
|
|
apply_deltas_osm2pgsql($apply);
|
|
}
|
|
else
|
|
{
|
|
apply_deltas_imposm3($apply);
|
|
}
|
|
update_state($dbh, $state);
|
|
$dbh->commit;
|
|
info("OSM database updated to version: ".$state->{timestamp});
|
|
if ($ENV{OSM_EXPIRE})
|
|
{
|
|
system("cat $dir/expire.list | render_expired --map=osm_carto --touch-from=11");
|
|
system("cat $dir/expire.list | render_expired --map=osm_bright --touch-from=11");
|
|
}
|
|
}
|
|
}
|
|
|
|
sub parse_geofabrik_state
|
|
{
|
|
my ($url_updates, $dir) = @_;
|
|
system("curl -s -f '$url_updates/state.txt' -o $dir/state.txt");
|
|
if (!-r "$dir/state.txt")
|
|
{
|
|
fatal("Error downloading $url_updates/state.txt");
|
|
}
|
|
else
|
|
{
|
|
my $state;
|
|
if (open FD, "<$dir/state.txt")
|
|
{
|
|
local $/ = undef;
|
|
$state = <FD>;
|
|
close FD;
|
|
$state = { map { (split /\s*=\s*/, $_, 2) } grep { !/^\s*(#.*)?$/so && /=/so } split /\n/, $state };
|
|
if (!$state->{timestamp} || !$state->{sequenceNumber})
|
|
{
|
|
fatal("State file incorrect, should have timestamp=<ISO8601 date> and sequenceNumber=<integer>");
|
|
}
|
|
$state->{timestamp} =~ s/\\//g;
|
|
return $state;
|
|
}
|
|
}
|
|
fatal("Error downloading $url_updates/state.txt");
|
|
}
|
|
|
|
sub load_geofabrik_deltas
|
|
{
|
|
my ($version, $state, $url_updates, $dir) = @_;
|
|
my ($timestamp, $i) = split /\s+/, $version;
|
|
my $apply = [];
|
|
while ($i <= $state->{sequenceNumber})
|
|
{
|
|
my $subdir = sprintf("%03d/%03d", $i / 1000000, ($i / 1000) % 1000);
|
|
my $fn = sprintf("%03d.osc.gz", $i % 1000);
|
|
info("Downloading $url_updates/$subdir/$fn");
|
|
system("mkdir -p $dir/$subdir && curl -C - -s -f '$url_updates/$subdir/$fn' -o $dir/$subdir/$fn");
|
|
if (-e "$dir/$subdir/$fn")
|
|
{
|
|
push @$apply, "$subdir/$fn";
|
|
}
|
|
else
|
|
{
|
|
fatal("Delta not available: $url_updates/$subdir/$fn\n");
|
|
}
|
|
$i++;
|
|
}
|
|
return $apply;
|
|
}
|
|
|
|
sub init_state
|
|
{
|
|
my ($dbh, $state) = @_;
|
|
$dbh->do('CREATE TABLE IF NOT EXISTS replication_state (name varchar(1024) not null primary key, value text not null)');
|
|
$dbh->do(
|
|
'INSERT INTO replication_state (name, value) VALUES (?, ?)',
|
|
{}, 'osm_version', $state->{timestamp} . ' ' . $state->{sequenceNumber}
|
|
);
|
|
}
|
|
|
|
sub update_state
|
|
{
|
|
my ($dbh, $state) = @_;
|
|
$dbh->do(
|
|
'UPDATE replication_state SET value=? WHERE name=?',
|
|
{}, $state->{timestamp} . ' ' . $state->{sequenceNumber}, 'osm_version'
|
|
);
|
|
}
|
|
|
|
sub init_osm2pgsql
|
|
{
|
|
my ($dbh, $path) = @_;
|
|
my $carto_dir = '/usr/share/mapnik/openstreetmap-carto-'.$ENV{OSM_CARTO_VERSION};
|
|
if (!$ENV{OSM_CARTO_VERSION})
|
|
{
|
|
fatal("osm-carto path not specified");
|
|
}
|
|
my $cmd =
|
|
"PGPASSWORD='".$ENV{PG_ENV_OSM_PASSWORD}."' osm2pgsql -I -s -c --hstore".
|
|
" --style $carto_dir/openstreetmap-carto.style".
|
|
" --tag-transform-script $carto_dir/openstreetmap-carto.lua".
|
|
" -C 4000 -G -H '".$ENV{PG_ENV_OSM_HOST}."' -U '".$ENV{PG_ENV_OSM_USER}."' -d '".$ENV{PG_ENV_OSM_DB}."'".
|
|
" -P ".($ENV{PG_ENV_OSM_PORT} || 5432)." '$path'";
|
|
system($cmd);
|
|
if ($?)
|
|
{
|
|
fatal("$cmd failed");
|
|
}
|
|
local $/ = undef;
|
|
my $fd;
|
|
open $fd, "$carto_dir/indexes.sql";
|
|
for my $index (split /;\s*/, <$fd>)
|
|
{
|
|
$dbh->do($index);
|
|
}
|
|
close $fd;
|
|
}
|
|
|
|
sub init_imposm3
|
|
{
|
|
my ($dbh, $path) = @_;
|
|
my $cmd =
|
|
"imposm3 import -connection 'postgis://".$ENV{PG_ENV_OSM_USER}.":".$ENV{PG_ENV_OSM_PASSWORD}.
|
|
"@".$ENV{PG_ENV_OSM_HOST}.(($ENV{PG_ENV_OSM_PORT}||5432) != 5432 ? ":".$ENV{PG_ENV_OSM_PORT} : "").
|
|
"/".$ENV{PG_ENV_OSM_DB}."' -cachedir '".$ENV{OSM_CACHE_DIR}."/imposm3-cache' -mapping '/home/imposm3-all.yml' -srid 4326 -diff".
|
|
" -read '$path' -write";
|
|
system($cmd);
|
|
if ($?)
|
|
{
|
|
fatal("$cmd failed");
|
|
}
|
|
my $indexes = "SET SEARCH_PATH TO import, public;
|
|
CREATE INDEX IF NOT EXISTS osm_polygon_area ON osm_polygon (st_area(geometry));
|
|
CREATE INDEX IF NOT EXISTS osm_point_tags ON osm_point USING gin (tags);
|
|
CREATE INDEX IF NOT EXISTS osm_linestring_tags ON osm_linestring USING gin (tags);
|
|
CREATE INDEX IF NOT EXISTS osm_polygon_tags ON osm_polygon USING gin (tags);
|
|
CREATE INDEX IF NOT EXISTS osm_relation_tags ON osm_relation USING gin (tags);
|
|
CREATE INDEX IF NOT EXISTS osm_point_text ON osm_point USING gin (to_tsvector('russian', tags::text));
|
|
CREATE INDEX IF NOT EXISTS osm_linestring_text ON osm_linestring USING gin (to_tsvector('russian', tags::text));
|
|
CREATE INDEX IF NOT EXISTS osm_polygon_text ON osm_polygon USING gin (to_tsvector('russian', tags::text));
|
|
CREATE INDEX IF NOT EXISTS osm_relation_text ON osm_relation USING gin (to_tsvector('russian', tags::text))";
|
|
foreach my $index (split /;\n/, $indexes)
|
|
{
|
|
$dbh->do($index);
|
|
}
|
|
}
|
|
|
|
sub apply_deltas_osm2pgsql
|
|
{
|
|
my ($apply, $carto_dir) = @_;
|
|
if (@$apply)
|
|
{
|
|
my $cmd =
|
|
"PGPASSWORD='".$ENV{PG_ENV_OSM_PASSWORD}."' osm2pgsql --append -e15 -o '".$ENV{OSM_CACHE_DIR}."/expire.list' -I -s --hstore".
|
|
" --style $carto_dir/openstreetmap-carto.style".
|
|
" --tag-transform-script $carto_dir/openstreetmap-carto.lua".
|
|
" -C 4000 -G -H '".$ENV{PG_ENV_OSM_HOST}."' -U '".$ENV{PG_ENV_OSM_USER}."' -d '".$ENV{PG_ENV_OSM_DB}."'".
|
|
" -P ".($ENV{PG_ENV_OSM_PORT} || 5432)." '".join("' '", @$apply)."'";
|
|
system($cmd);
|
|
if ($?)
|
|
{
|
|
fatal("$cmd failed");
|
|
}
|
|
}
|
|
}
|
|
|
|
sub apply_deltas_imposm3
|
|
{
|
|
my ($apply) = @_;
|
|
my $carto_dir = '/usr/share/mapnik/openstreetmap-carto-'.$ENV{OSM_CARTO_VERSION};
|
|
if (!$ENV{OSM_CARTO_VERSION})
|
|
{
|
|
fatal("osm-carto path not specified");
|
|
}
|
|
if (@$apply)
|
|
{
|
|
my $cmd =
|
|
"imposm3 diff -connection 'postgis://".$ENV{PG_ENV_OSM_USER}.":".$ENV{PG_ENV_OSM_PASSWORD}.
|
|
"@".$ENV{PG_ENV_OSM_HOST}.(($ENV{PG_ENV_OSM_PORT}||5432) != 5432 ? ":".$ENV{PG_ENV_OSM_PORT} : "").
|
|
"/".$ENV{PG_ENV_OSM_DB}."' -cachedir '".$ENV{OSM_CACHE_DIR}."/imposm3-cache' -mapping '/home/imposm3-all.yml' -srid 4326".
|
|
" '".join("' '", @$apply)."'";
|
|
system($cmd);
|
|
if ($?)
|
|
{
|
|
fatal("$cmd failed");
|
|
}
|
|
}
|
|
}
|
|
|
|
sub info
|
|
{
|
|
my ($msg) = @_;
|
|
print POSIX::strftime("%Y-%m-%d %H:%M:%S", localtime)." [info] $msg\n";
|
|
}
|
|
|
|
sub fatal
|
|
{
|
|
my ($msg) = @_;
|
|
eval { $dbh->rollback } if $dbh;
|
|
print POSIX::strftime("%Y-%m-%d %H:%M:%S", localtime)." [error] $msg\n";
|
|
exit(1);
|
|
}
|