Контейнер с патченым imposm3 и скриптом загрузки

master
Vitaliy Filippov 2018-12-08 17:34:16 +03:00
commit 2ecaeee50a
6 changed files with 640 additions and 0 deletions

36
Dockerfile Normal file
View File

@ -0,0 +1,36 @@
FROM debian:sid AS build
MAINTAINER Vitaliy Filippov <vitalif@mail.ru>
ADD etc /etc
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -q \
less wget sudo curl unzip git mc ca-certificates \
build-essential golang-go libleveldb-dev libgeos-dev && \
apt-get clean && rm -rf /var/lib/apt/lists/*
ADD split-ways.diff /root/
# Специальный патченый imposm3
RUN cd /root && git clone https://github.com/omniscale/imposm3/ go/src/github.com/omniscale/imposm3 && \
cd go/src/github.com/omniscale/imposm3 && git checkout c87a9a2b5761787a637e0fa73df9d4331846587d && \
patch -p1 < /root/split-ways.diff && make build
RUN dpkg -l | grep libgeos | awk '{print $2}' >> /root/pkg.lst
FROM debian:sid AS run
MAINTAINER Vitaliy Filippov <vitalif@mail.ru>
ADD etc /etc
COPY --from=build /root/pkg.lst /root/
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -q \
less wget sudo curl unzip git mc \
`cat /root/pkg.lst` libleveldb1v5 libdbd-pg-perl && \
apt-get clean && rm -rf /var/lib/apt/lists/*
COPY --from=build /root/go/src/github.com/omniscale/imposm3/imposm /usr/bin/imposm3
ADD home /home
CMD /home/osm_loader.pl

13
README.md Normal file
View File

@ -0,0 +1,13 @@
## Building
```
docker build -t imposm3 .
```
## Running
```
docker run -it -d --restart always \
-v /home/test/russia-latest.osm.pbf:/home/valhalla/data.osm.pbf \
--name valhalla -p 8002:8002 valhalla
```

4
etc/apt/apt.conf Normal file
View File

@ -0,0 +1,4 @@
APT::Install-Suggests false;
APT::Install-Recommends false;
Acquire::Retries 5;
Binary::apt::APT::Keep-Downloaded-Packages "false";

82
home/imposm3-all.yml Normal file
View File

@ -0,0 +1,82 @@
# Нужно применять с патченым imposm3 (патч тут же, imposm3-split-ways.diff)
# Иначе, несмотря на area_tags и linear_tags, здания оказываются в linestring
areas:
area_tags: [building, landuse, leisure, natural, aeroway]
linear_tags: [highway, barrier]
tags:
load_all: true
exclude:
- created_by
- source
tables:
point:
type: point
fields:
- name: osm_id
type: id
- name: geometry
type: geometry
- name: tags
type: hstore_tags
mapping:
__any__:
- __any__
linestring:
type: linestring
fields:
- name: osm_id
type: id
- name: geometry
type: geometry
- name: tags
type: hstore_tags
mapping:
__any__:
- __any__
polygon:
type: polygon
fields:
- name: osm_id
type: id
- name: geometry
type: geometry
- name: tags
type: hstore_tags
mapping:
__any__:
- __any__
relation_member:
type: relation_member
fields:
- name: osm_id
type: id
- name: member
type: member_id
- name: index
type: member_index
- name: type
type: member_type
- name: role
type: member_role
mapping:
__any__:
- __any__
relation:
type: relation
fields:
- name: osm_id
type: id
- key: type
name: type
type: string
- key: name
name: name
type: string
- name: tags
type: hstore_tags
mapping:
__any__:
- __any__

283
home/osm-loader.pl Executable file
View File

@ -0,0 +1,283 @@
#!/usr/bin/perl
# Configuration in env: OSM_CACHE_DIR, OSM_CARTO_VERSION, URL_LATEST, URL_UPDATES,
# plus PG_ENV_OSM_{DB,HOST,PORT,USER,PASSWORD} like in README for the render server itself
use strict;
use DBI;
use POSIX;
my $dbh;
my $dir = $ENV{OSM_CACHE_DIR} || '/var/lib/mod_tile/downloads';
my $method = $ENV{OSM_METHOD} || 'osm2pgsql-carto';
my $url_latest = $ENV{URL_LATEST} || 'http://download.geofabrik.de/russia-latest.osm.pbf';
my $url_updates = $ENV{URL_UPDATES} || 'http://download.geofabrik.de/russia-updates';
if (!$ENV{PG_ENV_OSM_DB})
{
fatal("OSM database not specified");
}
info("Started OSM update for database: $ENV{PG_ENV_OSM_DB}");
if ($method ne 'osm2pgsql-carto' && $method ne 'imposm3-all')
{
fatal("Incorrect OSM update mode: $method");
}
-e $dir || mkdir($dir);
chdir $dir or fatal("Failed to chdir $dir");
eval { run_update() };
if ($@)
{
fatal("$@");
}
exit(0);
sub run_update
{
my $state = parse_geofabrik_state($url_updates, $dir);
my $dbh = DBI->connect(
'dbi:Pg:dbname='.$ENV{PG_ENV_OSM_DB}.';host='.$ENV{PG_ENV_OSM_HOST}.';port='.($ENV{PG_ENV_OSM_PORT}||5432),
$ENV{PG_ENV_OSM_USER}, $ENV{PG_ENV_OSM_PASSWORD}, {AutoCommit => 0, RaiseError => 1}
);
my ($version) = eval { $dbh->selectrow_array(
'SELECT value FROM replication_state WHERE name=? FOR UPDATE', {}, 'osm_version'
) };
if (!$version)
{
if (!$ENV{OSM_INIT})
{
fatal("Current OSM version missing, run with OSM_INIT=1 environment variable to initialize");
}
$dbh->rollback; # иначе postgresql пишет "statements until the end ignored"
# создаём таблицу и оставляем её заблокированной
init_state($dbh, $state);
$dbh->do('CREATE EXTENSION IF NOT EXISTS postgis');
$dbh->do('CREATE EXTENSION IF NOT EXISTS hstore');
# качаем дамп
my ($fn) = $url_latest =~ /([^\/]+)$/so;
$fn =~ s/^([^\.]+)/$1-$state->{timestamp}/;
info("Downloading $url_latest");
system("curl -s -C - -f '$url_latest' -o $dir/$fn");
if ($? || !-e "$dir/$fn")
{
fatal("Failed to download $url_latest");
}
if ($method eq 'osm2pgsql-carto')
{
init_osm2pgsql($dbh, $state->{timestamp} . ' ' . $state->{sequenceNumber}, "$dir/$fn");
}
else
{
init_imposm3($dbh, $state->{timestamp} . ' ' . $state->{sequenceNumber}, "$dir/$fn");
}
$dbh->commit;
info("OSM database initialized with version: ".$state->{timestamp});
}
else
{
my $apply = load_geofabrik_deltas($version, $state, $url_updates, $dir);
chdir($dir);
if ($method eq 'osm2pgsql-carto')
{
apply_deltas_osm2pgsql($apply);
}
else
{
apply_deltas_imposm3($apply);
}
update_state($dbh, $state);
$dbh->commit;
info("OSM database updated to version: ".$state->{timestamp});
if ($ENV{OSM_EXPIRE})
{
system("cat $dir/expire.list | render_expired --map=osm_carto --touch-from=11");
system("cat $dir/expire.list | render_expired --map=osm_bright --touch-from=11");
}
}
}
sub parse_geofabrik_state
{
my ($url_updates, $dir) = @_;
system("curl -s -f '$url_updates/state.txt' -o $dir/state.txt");
if (!-r "$dir/state.txt")
{
fatal("Error downloading $url_updates/state.txt");
}
else
{
my $state;
if (open FD, "<$dir/state.txt")
{
local $/ = undef;
$state = <FD>;
close FD;
$state = { map { (split /\s*=\s*/, $_, 2) } grep { !/^\s*(#.*)?$/so && /=/so } split /\n/, $state };
if (!$state->{timestamp} || !$state->{sequenceNumber})
{
fatal("State file incorrect, should have timestamp=<ISO8601 date> and sequenceNumber=<integer>");
}
$state->{timestamp} =~ s/\\//g;
return $state;
}
}
fatal("Error downloading $url_updates/state.txt");
}
sub load_geofabrik_deltas
{
my ($version, $state, $url_updates, $dir) = @_;
my ($timestamp, $i) = split /\s+/, $version;
my $apply = [];
while ($i <= $state->{sequenceNumber})
{
my $subdir = sprintf("%03d/%03d", $i / 1000000, ($i / 1000) % 1000);
my $fn = sprintf("%03d.osc.gz", $i % 1000);
info("Downloading $url_updates/$subdir/$fn");
system("mkdir -p $dir/$subdir && curl -C - -s -f '$url_updates/$subdir/$fn' -o $dir/$subdir/$fn");
if (-e "$dir/$subdir/$fn")
{
push @$apply, "$subdir/$fn";
}
else
{
fatal("Delta not available: $url_updates/$subdir/$fn\n");
}
$i++;
}
return $apply;
}
sub init_state
{
my ($dbh, $state) = @_;
$dbh->do('CREATE TABLE IF NOT EXISTS replication_state (name varchar(1024) not null primary key, value text not null)');
$dbh->do(
'INSERT INTO replication_state (name, value) VALUES (?, ?)',
{}, 'osm_version', $state->{timestamp} . ' ' . $state->{sequenceNumber}
);
}
sub update_state
{
my ($dbh, $state) = @_;
$dbh->do(
'UPDATE replication_state SET value=? WHERE name=?',
{}, $state->{timestamp} . ' ' . $state->{sequenceNumber}, 'osm_version'
);
}
sub init_osm2pgsql
{
my ($dbh, $path) = @_;
my $carto_dir = '/usr/share/mapnik/openstreetmap-carto-'.$ENV{OSM_CARTO_VERSION};
if (!$ENV{OSM_CARTO_VERSION})
{
fatal("osm-carto path not specified");
}
my $cmd =
"PGPASSWORD='".$ENV{PG_ENV_OSM_PASSWORD}."' osm2pgsql -I -s -c --hstore".
" --style $carto_dir/openstreetmap-carto.style".
" --tag-transform-script $carto_dir/openstreetmap-carto.lua".
" -C 4000 -G -H '".$ENV{PG_ENV_OSM_HOST}."' -U '".$ENV{PG_ENV_OSM_USER}."' -d '".$ENV{PG_ENV_OSM_DB}."'".
" -P ".($ENV{PG_ENV_OSM_PORT} || 5432)." '$path'";
system($cmd);
if ($?)
{
fatal("$cmd failed");
}
local $/ = undef;
my $fd;
open $fd, "$carto_dir/indexes.sql";
for my $index (split /;\s*/, <$fd>)
{
$dbh->do($index);
}
close $fd;
}
sub init_imposm3
{
my ($dbh, $path) = @_;
my $cmd =
"imposm3 import -connection 'postgis://".$ENV{PG_ENV_OSM_USER}.":".$ENV{PG_ENV_OSM_PASSWORD}.
"@".$ENV{PG_ENV_OSM_HOST}.(($ENV{PG_ENV_OSM_PORT}||5432) != 5432 ? ":".$ENV{PG_ENV_OSM_PORT} : "").
"/".$ENV{PG_ENV_OSM_DB}."' -cachedir '".$ENV{OSM_CACHE_DIR}."/imposm3-cache' -mapping '/home/imposm3-all.yml' -srid 4326 -diff".
" -read '$path' -write";
system($cmd);
if ($?)
{
fatal("$cmd failed");
}
my $indexes = "SET SEARCH_PATH TO import, public;
CREATE INDEX IF NOT EXISTS osm_polygon_area ON osm_polygon (st_area(geometry));
CREATE INDEX IF NOT EXISTS osm_point_tags ON osm_point USING gin (tags);
CREATE INDEX IF NOT EXISTS osm_linestring_tags ON osm_linestring USING gin (tags);
CREATE INDEX IF NOT EXISTS osm_polygon_tags ON osm_polygon USING gin (tags);
CREATE INDEX IF NOT EXISTS osm_relation_tags ON osm_relation USING gin (tags);
CREATE INDEX IF NOT EXISTS osm_point_text ON osm_point USING gin (to_tsvector('russian', tags::text));
CREATE INDEX IF NOT EXISTS osm_linestring_text ON osm_linestring USING gin (to_tsvector('russian', tags::text));
CREATE INDEX IF NOT EXISTS osm_polygon_text ON osm_polygon USING gin (to_tsvector('russian', tags::text));
CREATE INDEX IF NOT EXISTS osm_relation_text ON osm_relation USING gin (to_tsvector('russian', tags::text))";
foreach my $index (split /;\n/, $indexes)
{
$dbh->do($index);
}
}
sub apply_deltas_osm2pgsql
{
my ($apply, $carto_dir) = @_;
if (@$apply)
{
my $cmd =
"PGPASSWORD='".$ENV{PG_ENV_OSM_PASSWORD}."' osm2pgsql --append -e15 -o '".$ENV{OSM_CACHE_DIR}."/expire.list' -I -s --hstore".
" --style $carto_dir/openstreetmap-carto.style".
" --tag-transform-script $carto_dir/openstreetmap-carto.lua".
" -C 4000 -G -H '".$ENV{PG_ENV_OSM_HOST}."' -U '".$ENV{PG_ENV_OSM_USER}."' -d '".$ENV{PG_ENV_OSM_DB}."'".
" -P ".($ENV{PG_ENV_OSM_PORT} || 5432)." '".join("' '", @$apply)."'";
system($cmd);
if ($?)
{
fatal("$cmd failed");
}
}
}
sub apply_deltas_imposm3
{
my ($apply) = @_;
my $carto_dir = '/usr/share/mapnik/openstreetmap-carto-'.$ENV{OSM_CARTO_VERSION};
if (!$ENV{OSM_CARTO_VERSION})
{
fatal("osm-carto path not specified");
}
if (@$apply)
{
my $cmd =
"imposm3 diff -connection 'postgis://".$ENV{PG_ENV_OSM_USER}.":".$ENV{PG_ENV_OSM_PASSWORD}.
"@".$ENV{PG_ENV_OSM_HOST}.(($ENV{PG_ENV_OSM_PORT}||5432) != 5432 ? ":".$ENV{PG_ENV_OSM_PORT} : "").
"/".$ENV{PG_ENV_OSM_DB}."' -cachedir '".$ENV{OSM_CACHE_DIR}."/imposm3-cache' -mapping '/home/imposm3-all.yml' -srid 4326".
" '".join("' '", @$apply)."'";
system($cmd);
if ($?)
{
fatal("$cmd failed");
}
}
}
sub info
{
my ($msg) = @_;
print POSIX::strftime("%Y-%m-%d %H:%M:%S", localtime)." [info] $msg\n";
}
sub fatal
{
my ($msg) = @_;
eval { $dbh->rollback } if $dbh;
print POSIX::strftime("%Y-%m-%d %H:%M:%S", localtime)." [error] $msg\n";
exit(1);
}

222
split-ways.diff Normal file
View File

@ -0,0 +1,222 @@
commit 8e1da67b3d4221aa16469cba53c8097ddbe78cb8
Author: Vitaliy Filippov <vitalif@yourcmc.ru>
Date: Sat Sep 29 02:47:19 2018 +0300
Split ways between linestrings / polygons correctly (so one object cannot be a linestring and a polygon at the same time)
diff --git a/mapping/mapping.go b/mapping/mapping.go
index 4f48658..7e58a93 100644
--- a/mapping/mapping.go
+++ b/mapping/mapping.go
@@ -275,7 +275,7 @@ type elementFilter func(tags element.Tags, key Key, closed bool) bool
type tableElementFilters map[string][]elementFilter
-func (m *Mapping) addTypedFilters(tableType TableType, filters tableElementFilters) {
+func (m *Mapping) getAreaTags() (map[Key]struct{}, map[Key]struct{}) {
var areaTags map[Key]struct{}
var linearTags map[Key]struct{}
if m.Conf.Areas.AreaTags != nil {
@@ -290,42 +290,7 @@ func (m *Mapping) addTypedFilters(tableType TableType, filters tableElementFilte
linearTags[Key(tag)] = struct{}{}
}
}
-
- for name, t := range m.Conf.Tables {
- if TableType(t.Type) != GeometryTable && TableType(t.Type) != tableType {
- continue
- }
- if TableType(t.Type) == LineStringTable && areaTags != nil {
- f := func(tags element.Tags, key Key, closed bool) bool {
- if closed {
- if tags["area"] == "yes" {
- return false
- }
- if tags["area"] != "no" {
- if _, ok := areaTags[key]; ok {
- return false
- }
- }
- }
- return true
- }
- filters[name] = append(filters[name], f)
- }
- if TableType(t.Type) == PolygonTable && linearTags != nil {
- f := func(tags element.Tags, key Key, closed bool) bool {
- if closed && tags["area"] == "no" {
- return false
- }
- if tags["area"] != "yes" {
- if _, ok := linearTags[key]; ok {
- return false
- }
- }
- return true
- }
- filters[name] = append(filters[name], f)
- }
- }
+ return areaTags, linearTags
}
func (m *Mapping) addRelationFilters(tableType TableType, filters tableElementFilters) {
diff --git a/mapping/matcher.go b/mapping/matcher.go
index 2c58446..346782c 100644
--- a/mapping/matcher.go
+++ b/mapping/matcher.go
@@ -10,13 +10,11 @@ func (m *Mapping) pointMatcher() (NodeMatcher, error) {
m.mappings(PointTable, mappings)
filters := make(tableElementFilters)
m.addFilters(filters)
- m.addTypedFilters(PointTable, filters)
tables, err := m.tables(PointTable)
return &tagMatcher{
mappings: mappings,
filters: filters,
tables: tables,
- matchAreas: false,
}, err
}
@@ -25,12 +23,15 @@ func (m *Mapping) lineStringMatcher() (WayMatcher, error) {
m.mappings(LineStringTable, mappings)
filters := make(tableElementFilters)
m.addFilters(filters)
- m.addTypedFilters(LineStringTable, filters)
+ areaTags, linearTags := m.getAreaTags()
tables, err := m.tables(LineStringTable)
return &tagMatcher{
mappings: mappings,
filters: filters,
tables: tables,
+ areaTags: areaTags,
+ linearTags: linearTags,
+ matchLines: true,
matchAreas: false,
}, err
}
@@ -40,7 +41,7 @@ func (m *Mapping) polygonMatcher() (RelWayMatcher, error) {
m.mappings(PolygonTable, mappings)
filters := make(tableElementFilters)
m.addFilters(filters)
- m.addTypedFilters(PolygonTable, filters)
+ areaTags, linearTags := m.getAreaTags()
relFilters := make(tableElementFilters)
m.addRelationFilters(PolygonTable, relFilters)
tables, err := m.tables(PolygonTable)
@@ -49,6 +50,9 @@ func (m *Mapping) polygonMatcher() (RelWayMatcher, error) {
filters: filters,
tables: tables,
relFilters: relFilters,
+ areaTags: areaTags,
+ linearTags: linearTags,
+ matchLines: false,
matchAreas: true,
}, err
}
@@ -58,8 +62,6 @@ func (m *Mapping) relationMatcher() (RelationMatcher, error) {
m.mappings(RelationTable, mappings)
filters := make(tableElementFilters)
m.addFilters(filters)
- m.addTypedFilters(PolygonTable, filters)
- m.addTypedFilters(RelationTable, filters)
relFilters := make(tableElementFilters)
m.addRelationFilters(RelationTable, relFilters)
tables, err := m.tables(RelationTable)
@@ -68,7 +70,6 @@ func (m *Mapping) relationMatcher() (RelationMatcher, error) {
filters: filters,
tables: tables,
relFilters: relFilters,
- matchAreas: true,
}, err
}
@@ -77,7 +78,6 @@ func (m *Mapping) relationMemberMatcher() (RelationMatcher, error) {
m.mappings(RelationMemberTable, mappings)
filters := make(tableElementFilters)
m.addFilters(filters)
- m.addTypedFilters(RelationMemberTable, filters)
relFilters := make(tableElementFilters)
m.addRelationFilters(RelationMemberTable, relFilters)
tables, err := m.tables(RelationMemberTable)
@@ -86,7 +86,6 @@ func (m *Mapping) relationMemberMatcher() (RelationMatcher, error) {
filters: filters,
tables: tables,
relFilters: relFilters,
- matchAreas: true,
}, err
}
@@ -127,6 +126,9 @@ type tagMatcher struct {
tables map[string]*rowBuilder
filters tableElementFilters
relFilters tableElementFilters
+ areaTags map[Key]struct{}
+ linearTags map[Key]struct{}
+ matchLines bool
matchAreas bool
}
@@ -135,23 +137,7 @@ func (tm *tagMatcher) MatchNode(node *element.Node) []Match {
}
func (tm *tagMatcher) MatchWay(way *element.Way) []Match {
- if tm.matchAreas { // match way as polygon
- if way.IsClosed() {
- if way.Tags["area"] == "no" {
- return nil
- }
- return tm.match(way.Tags, true, false)
- }
- } else { // match way as linestring
- if way.IsClosed() {
- if way.Tags["area"] == "yes" {
- return nil
- }
- return tm.match(way.Tags, true, false)
- }
- return tm.match(way.Tags, false, false)
- }
- return nil
+ return tm.match(way.Tags, way.IsClosed(), false)
}
func (tm *tagMatcher) MatchRelation(rel *element.Relation) []Match {
@@ -205,7 +191,35 @@ func (tm *tagMatcher) match(tags element.Tags, closed bool, relation bool) []Mat
for t, match := range tables {
filters, ok := tm.filters[t.Name]
filteredOut := false
- if ok {
+ if !relation && (tm.matchLines || tm.matchAreas) {
+ if !closed {
+ // open way is always a linestring
+ filteredOut = tm.matchLines == false
+ } else {
+ // allow to include closed ways as linestrings if explicitly marked
+ // default -> area
+ // area=no or linear tags -> line
+ // area=yes or area tags -> area
+ filteredOut = tm.matchAreas && tags["area"] == "no" || !tm.matchAreas && tags["area"] != "no"
+ for k, _ := range tm.linearTags {
+ if _, ok := tags[string(k)]; ok {
+ filteredOut = tm.matchAreas
+ break
+ }
+ }
+ // but area=yes or area tag means it shouldn't be a linestring
+ if tags["area"] == "yes" {
+ filteredOut = tm.matchAreas
+ }
+ for k, _ := range tm.areaTags {
+ if _, ok := tags[string(k)]; ok {
+ filteredOut = !tm.matchAreas
+ break
+ }
+ }
+ }
+ }
+ if ok && !filteredOut {
for _, filter := range filters {
if !filter(tags, Key(match.Key), closed) {
filteredOut = true