travis: deploy releases to github with custom leveldb and geos

.travis.yml

@@ -1,34 +1,132 @@
+dist: trusty
+sudo: false
+
 language: go
 
 go:
- - 1.8.5
+  - "1.7.x"
- - 1.9.2
+  - "1.9.x"
- - tip
+  - "1.10.x" # we use this for releases, see deploy.on.condition below
+  - tip
+
+matrix:
+  allow_failures:
+    - go: tip
+      fast_finish: true
+
+env:
+  global:
+    - GEOS_VERSION=3.6.2
+    - PROTOBUF_VERSION=3.5.1
+    - LEVELDB_VERSION=1.21 # TODO: update before_script when 1.21 is released
+    # NOTE increase DEPS_REVISION everytime one of the compiled deps changes!
+    - DEPS_REVISION=1
+    # PREFIX is where we install our compiled deps
+    - PREFIX=${HOME}/local-${DEPS_REVISION}
+
+    # tell Go to find our compiled deps
+    - CGO_CFLAGS=-I${PREFIX}/include
+    - CGO_LDFLAGS=-L${PREFIX}/lib
+    - LD_LIBRARY_PATH=${PREFIX}/lib
+
+    # PKG_DIR is where we assemble the final package for releases
+    - PKG_DIR=/tmp/imposm_packaging
+
+    - PGUSER=postgres
+
+    # enable ccache by updating PATH
+    - PATH=/usr/lib/ccache:$PATH
+    - CCACHE_TEMPDIR=/tmp/.ccache-temp
+
+cache:
+  directories:
+    - ${HOME}/.ccache
+    - ${PREFIX}
 
 addons:
     postgresql: "9.5"
     apt:
+      # List of whitelisted in travis packages for ubuntu-trusty can be found here:
+      #   https://github.com/travis-ci/apt-package-whitelist/blob/master/ubuntu-trusty
       packages:
         - postgresql-9.5-postgis-2.3
-        - libgeos-dev
+        - chrpath
-        - libprotobuf-dev
-        - protobuf-compiler
-        - osmosis
 
 before_script:
   - psql -U postgres -c 'create database travis' || true
   - psql -U postgres -c "create extension if not exists postgis"
   - psql -U postgres -c "create extension if not exists hstore"
 
-env:
-  global:
-    - PGUSER=postgres
 
-install:
+before_install:
-  - LEVELDB_VER=1.18
+  - ccache --show-stats
-  - curl -fsSL https://github.com/google/leveldb/archive/v${LEVELDB_VER}.tar.gz | tar -C ${HOME} -xzf -
+
-  - make -C ${HOME}/leveldb-${LEVELDB_VER} -j2
+  # build GEOS
-  - sudo mv ${HOME}/leveldb-${LEVELDB_VER}/libleveldb* /usr/lib
+  - |
-  - sudo cp -a ${HOME}/leveldb-${LEVELDB_VER}/include/leveldb /usr/include
+    if [[ ! -e ${PREFIX}/lib/libgeos_c.so ]]; then
+      curl -fsSL http://download.osgeo.org/geos/geos-${GEOS_VERSION}.tar.bz2 | \
+        tar -jxf - -C ${HOME}/build
+      cd ${HOME}/build/geos-${GEOS_VERSION}
+      ./configure --prefix=$PREFIX
+      make -j2
+      make install
+    fi
+
+  # build LevelDB
+  - |
+    if [[ ! -e ${PREFIX}/lib/libleveldb.so ]]; then
+      mkdir -p ${HOME}/build/leveldb
+      curl -fsSL https://github.com/google/leveldb/archive/master.tar.gz | \
+        tar -zxf - --strip-components 1 -C ${HOME}/build/leveldb
+      mkdir -p ${HOME}/build/leveldb/build
+      cd ${HOME}/build/leveldb/build
+      cmake .. -DBUILD_SHARED_LIBS=1 && make -j 2
+      mkdir -p ${PREFIX}/{lib,include}
+      cp -R ${HOME}/build/leveldb/build/liblevel* ${PREFIX}/lib/
+      cp -R ${HOME}/build/leveldb/include/leveldb ${PREFIX}/include/
+    fi
+
+  - ccache --show-stats
+
+script:
+  - cd ${TRAVIS_BUILD_DIR}
+  - LEVELDB_POST_121=1 make
+
+before_deploy:
+  - cd ${TRAVIS_BUILD_DIR}
+  - mkdir -p ${PKG_DIR}/lib
+
+  # copy files
+  - cp imposm3 ${PKG_DIR}
+  - cp example-mapping.json ${PKG_DIR}/mapping.json
+
+  # copy/link libs
+  - cd ${PREFIX}/lib
+  - cp libgeos_c.so ${PKG_DIR}/lib
+  - ln -s libgeos_c.so ${PKG_DIR}/lib/libgeos_c.so.1
+  - cp libgeos.so ${PKG_DIR}/lib
+  - ln -s libgeos.so ${PKG_DIR}/lib/libgeos-${GEOS_VERSION}.so
+  - cp -R libleveldb.so* ${PKG_DIR}/lib
+
+  # update RPATH of libgeos_c.so so that is points to our libgeos.so
+  - chrpath ${PKG_DIR}/lib/libgeos_c.so -r '${ORIGIN}'
+
+  # create tar.gz
+  - BUILD_VERSION=`${PKG_DIR}/imposm3 version`-linux-x86-64
+  - mv ${PKG_DIR} /tmp/imposm3-${BUILD_VERSION}
+  - cd /tmp && tar zcvf imposm3-${BUILD_VERSION}.tar.gz imposm3-${BUILD_VERSION}
+
+  # move back to build dir for deploy
+  - cd ${TRAVIS_BUILD_DIR}
+
+deploy:
+  provider: releases
+  api_key:
+    secure: fBhtCy6vdxxxuLYuJWXeTHJb3SAZCCbUJncIvR9ZOLCDWMJAPBnzPoqJFbByWzl6XghVRwhy9fe/82vzqByKiB/AQpqGqBxlwA9dSsqvSQcQxomAoHKKfZVdOuxM2bLDW3v5pJpmFtCMwhWjgwIOb9WxnBVeLxBbsq2Ox49tgHw=
+  file: /tmp/imposm3-${BUILD_VERSION}.tar.gz
+  skip_cleanup: true
+  on:
+    repo: omniscale/imposm3
+    tags: true
+    condition: $TRAVIS_GO_VERSION =~ ^1\.10(\.[0-9]+)?$
 
-script: make

Makefile

@@ -17,10 +17,15 @@ endif
 
 BUILD_DATE=$(shell date +%Y%m%d)
 BUILD_REV=$(shell git rev-parse --short HEAD)
-ifndef IMPOSM_BUILD_RELEASE
+BUILD_BRANCH=$(shell git rev-parse --abbrev-ref HEAD)
-BUILD_VERSION=dev-$(BUILD_DATE)-$(BUILD_REV)
+TAG=$(shell git name-rev --tags --name-only $(BUILD_REV))
+ifeq ($(TAG),undefined)
+	BUILD_VERSION=$(BUILD_BRANCH)-$(BUILD_DATE)-$(BUILD_REV)
+else
+	# use TAG but strip v of v1.2.3
+	BUILD_VERSION=$(TAG:v%=%)
 endif
-VERSION_LDFLAGS=-X github.com/omniscale/imposm3.buildVersion=$(BUILD_VERSION)
+VERSION_LDFLAGS=-X github.com/omniscale/imposm3.Version=$(BUILD_VERSION)
 
 all: build test
 

test/Makefile

@@ -9,17 +9,15 @@ endif
 all: test
 
 clean:
-	rm -rf build
+	rm generated/*.{pbf,gz}
 
-PBF_FILES=$(addprefix build/,$(patsubst %.osm,%.pbf,$(wildcard *.osm)))
+PBF_FILES=$(addprefix generated/,$(patsubst %.osm,%.pbf,$(wildcard *.osm)))
-OSCGZ_FILES=$(addprefix build/,$(patsubst %.osc,%.osc.gz,$(wildcard *.osc)))
+OSCGZ_FILES=$(addprefix generated/,$(patsubst %.osc,%.osc.gz,$(wildcard *.osc)))
 
-build/%.pbf: %.osm
+generated/%.pbf: %.osm
-	@mkdir -p build
 	osmosis --read-xml $< --sort type="TypeThenId" --write-pbf $@ omitmetadata=true
 
-build/%.osc.gz: %.osc
+generated/%.osc.gz: %.osc
-	@mkdir -p build
 	gzip --stdout $< > $@
 
 files: $(PBF_FILES) $(OSCGZ_FILES)

test/any_any_test.go

@@ -20,7 +20,7 @@ func TestAnyAny_Prepare(t *testing.T) {
 	ts.config = importConfig{
 		connection:      "postgis://",
 		cacheDir:        ts.dir,
-		osmFileName:     "build/any_any.pbf",
+		osmFileName:     "generated/any_any.pbf",
 		mappingFileName: "any_any_mapping.json",
 	}
 	ts.g = geos.NewGeos()

test/completedb_test.go

@@ -29,7 +29,7 @@ func TestComplete_Prepare(t *testing.T) {
 	ts.config = importConfig{
 		connection:      "postgis://",
 		cacheDir:        ts.dir,
-		osmFileName:     "build/complete_db.pbf",
+		osmFileName:     "generated/complete_db.pbf",
 		mappingFileName: "complete_db_mapping.json",
 	}
 	ts.g = geos.NewGeos()
@@ -400,7 +400,7 @@ func TestComplete_HstoreTags(t *testing.T) {
 // #######################################################################
 
 func TestComplete_Update(t *testing.T) {
-	ts.updateOsm(t, "./build/complete_db.osc.gz")
+	ts.updateOsm(t, "./generated/complete_db.osc.gz")
 }
 
 // #######################################################################

test/expire_tiles_test.go

@@ -24,7 +24,7 @@ func TestExpireTiles_Prepare(t *testing.T) {
 	ts.config = importConfig{
 		connection:      "postgis://",
 		cacheDir:        ts.dir,
-		osmFileName:     "build/expire_tiles.pbf",
+		osmFileName:     "generated/expire_tiles.pbf",
 		mappingFileName: "expire_tiles_mapping.yml",
 		expireTileDir:   filepath.Join(ts.dir, "expiretiles"),
 	}
@@ -62,7 +62,7 @@ func TestExpireTiles_Elements(t *testing.T) {
 }
 
 func TestExpireTiles_Update(t *testing.T) {
-	ts.updateOsm(t, "build/expire_tiles.osc.gz")
+	ts.updateOsm(t, "generated/expire_tiles.osc.gz")
 }
 
 func TestExpireTiles_CheckExpireFile(t *testing.T) {

test/generated/README

@@ -0,0 +1,2 @@
+Files in this directory are generated by `make files` in ./test by calling osmosis and gzip.
+The sources for these files are in ./test.

test/route_relation_test.go

@@ -20,7 +20,7 @@ func TestRouteRelation_Prepare(t *testing.T) {
 	ts.config = importConfig{
 		connection:      "postgis://",
 		cacheDir:        ts.dir,
-		osmFileName:     "build/route_relation.pbf",
+		osmFileName:     "generated/route_relation.pbf",
 		mappingFileName: "route_relation_mapping.yml",
 	}
 	ts.g = geos.NewGeos()
@@ -107,7 +107,7 @@ func TestRouteRelation_NoRouteWithMissingMember(t *testing.T) {
 // #######################################################################
 
 func TestRouteRelation_Update(t *testing.T) {
-	ts.updateOsm(t, "./build/route_relation.osc.gz")
+	ts.updateOsm(t, "./generated/route_relation.osc.gz")
 }
 
 // #######################################################################

test/single_table_test.go

@@ -23,7 +23,7 @@ func TestSingleTable_Prepare(t *testing.T) {
 	ts.config = importConfig{
 		connection:      "postgis://",
 		cacheDir:        ts.dir,
-		osmFileName:     "build/single_table.pbf",
+		osmFileName:     "generated/single_table.pbf",
 		mappingFileName: "single_table_mapping.json",
 	}
 	ts.g = geos.NewGeos()
@@ -167,7 +167,7 @@ func TestSingleTable_DuplicateIds1(t *testing.T) {
 // #######################################################################
 
 func TestSingleTable_Update(t *testing.T) {
-	ts.updateOsm(t, "./build/single_table.osc.gz")
+	ts.updateOsm(t, "./generated/single_table.osc.gz")
 }
 
 // #######################################################################

version.go

@@ -1,12 +1,5 @@
 package imposm3
 
-var Version string
+// Version gets replaced while building with
-
+// go build -ldflags "-X github.com/omniscale/imposm3.Version 1.2.3"
-// buidVersion gets replaced while building with
+var Version = "0.0.0-dev"
-// go build -ldflags "-X github.com/omniscale/imposm3.buildVersion 1234"
-var buildVersion string
-
-func init() {
-	Version = "0.5.0"
-	Version += buildVersion
-}