Initial

6 files changed, 334 insertions, 0 deletions

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..2a550d9
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+.idea
+osm2mgo
\ No newline at end of file
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..c52dd76
--- /dev/null
+++ b/README.md
@@ -0,0 +1,22 @@
+# OpenStreetMaps to Mongo
+
+Simple loader from osm dump file to mongodb. Based on https://github.com/paulmach/osm package.
+
+## Build
+
+`go build -o osm2go`
+
+## Usage
+
+`./osm2go -osmfile PATH_TO_OSM_FILE [-dbconnection mongodb://localhost:27017] [-dbname osm]`
+
+* `osmfile` required, path to *.osm or *.osm.pbf file
+* `dbconnection` optional, mongodb connection string (default: `mongodb://localhost:27017`) 
+* `dbname` optional, mongodb database name (default: `osm`) 
+
+## Example
+
+```
+# ./osm2mgo -osmfile ~/Downloads/RU.pbf
+Nodes: 1294069 Ways: 0 Relations: 0
+```
\ No newline at end of file
diff --git a/go.mod b/go.mod
new file mode 100644
index 0000000..06c3118
--- /dev/null
+++ b/go.mod
@@ -0,0 +1,20 @@
+module osm2mongo
+
+go 1.12
+
+require (
+	github.com/go-stack/stack v1.8.0 // indirect
+	github.com/gogo/protobuf v1.2.1 // indirect
+	github.com/golang/snappy v0.0.1 // indirect
+	github.com/google/go-cmp v0.3.0 // indirect
+	github.com/paulmach/orb v0.1.3
+	github.com/paulmach/osm v0.0.1
+	github.com/stretchr/testify v1.3.0 // indirect
+	github.com/tidwall/pretty v0.0.0-20190325153808-1166b9ac2b65 // indirect
+	github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c // indirect
+	github.com/xdg/stringprep v1.0.0 // indirect
+	go.mongodb.org/mongo-driver v1.0.2
+	golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f // indirect
+	golang.org/x/sync v0.0.0-20190423024810-112230192c58 // indirect
+	golang.org/x/text v0.3.2 // indirect
+)
diff --git a/go.sum b/go.sum
new file mode 100644
index 0000000..419da2d
--- /dev/null
+++ b/go.sum
@@ -0,0 +1,42 @@
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
+github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/paulmach/orb v0.1.3 h1:Wa1nzU269Zv7V9paVEY1COWW8FCqv4PC/KJRbJSimpM=
+github.com/paulmach/orb v0.1.3/go.mod h1:VFlX/8C+IQ1p6FTRRKzKoOPJnvEtA5G0Veuqwbu//Vk=
+github.com/paulmach/osm v0.0.1 h1:TxU/uZnJ+ssntblY6kSieOP4tQv31VW8wfyf4L3BeKs=
+github.com/paulmach/osm v0.0.1/go.mod h1:d6Ez0X1es6TaiYBxNmn1WbutRRQleZdvpoqeWdH/5kM=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/tidwall/pretty v0.0.0-20190325153808-1166b9ac2b65 h1:rQ229MBgvW68s1/g6f1/63TgYwYxfF4E+bi/KC19P8g=
+github.com/tidwall/pretty v0.0.0-20190325153808-1166b9ac2b65/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk=
+github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
+github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0=
+github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
+go.mongodb.org/mongo-driver v1.0.2 h1:RwjK1tKt7VPqQh3tsjiEqKJg75GNhP/loch+PwRc4ig=
+go.mongodb.org/mongo-driver v1.0.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f h1:R423Cnkcp5JABoeemiGEPlt9tHXFfw5kvc0yqlxRPWo=
+golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
diff --git a/main.go b/main.go
new file mode 100644
index 0000000..1458fe8
--- /dev/null
+++ b/main.go
@@ -0,0 +1,191 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"time"
+
+	"github.com/paulmach/osm"
+	"github.com/paulmach/osm/osmpbf"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+	"go.mongodb.org/mongo-driver/x/bsonx"
+)
+
+func main() {
+	dbconnection := flag.String("dbconnection", "mongodb://localhost:27017", "Mongo database name")
+	dbname := flag.String("dbname", "osm", "Mongo database name")
+	osmfile := flag.String("osmfile", "", "OSM file")
+	flag.Parse()
+	ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
+	client, err := mongo.Connect(ctx, options.Client().ApplyURI(*dbconnection))
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer client.Disconnect(context.Background())
+	db := client.Database(*dbname)
+	if err := read(db, *osmfile); err != nil {
+		log.Fatal(err)
+	}
+
+}
+
+func read(db *mongo.Database, file string) error {
+	nodes := db.Collection("nodes")
+	_, _ = nodes.Indexes().CreateOne(
+		context.Background(),
+		mongo.IndexModel{
+			Keys:    bsonx.Doc{{"osm_id", bsonx.Int32(1)}},
+			Options: options.Index().SetUnique(true).SetSparse(true),
+		},
+	)
+	_, _ = nodes.Indexes().CreateOne(
+		context.Background(),
+		mongo.IndexModel{
+			Keys:    bsonx.Doc{{"coords", bsonx.Int32(1)}},
+			Options: options.Index().SetSphereVersion(2).SetSparse(true),
+		},
+	)
+
+	ways := db.Collection("ways")
+	_, _ = ways.Indexes().CreateOne(
+		context.Background(),
+		mongo.IndexModel{
+			Keys:    bsonx.Doc{{"osm_id", bsonx.Int32(1)}},
+			Options: options.Index().SetUnique(true).SetSparse(true),
+		},
+	)
+	_, _ = ways.Indexes().CreateOne(
+		context.Background(),
+		mongo.IndexModel{
+			Keys:    bsonx.Doc{{"nodes", bsonx.Int32(1)}},
+			Options: options.Index().SetSparse(true),
+		},
+	)
+
+	relations := db.Collection("relations")
+	_, _ = nodes.Indexes().CreateOne(
+		context.Background(),
+		mongo.IndexModel{
+			Keys:    bsonx.Doc{{"osm_id", bsonx.Int32(1)}},
+			Options: options.Index().SetUnique(true).SetSparse(true),
+		},
+	)
+	_, _ = nodes.Indexes().CreateOne(
+		context.Background(),
+		mongo.IndexModel{
+			Keys:    bsonx.Doc{{"members.ref", bsonx.Int32(1)}},
+			Options: options.Index().SetUnique(true).SetSparse(true),
+		},
+	)
+	_, _ = nodes.Indexes().CreateOne(
+		context.Background(),
+		mongo.IndexModel{
+			Keys:    bsonx.Doc{{"members.coords", bsonx.Int32(1)}},
+			Options: options.Index().SetSphereVersion(2).SetSparse(true),
+		},
+	)
+
+	f, err := os.Open(file)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	opts := (new(options.ReplaceOptions)).SetUpsert(true)
+	nc := 0
+	wc := 0
+	rc := 0
+
+	scanner := osmpbf.New(context.Background(), f, 3)
+	defer scanner.Close()
+
+	for scanner.Scan() {
+		o := scanner.Object()
+		switch o := o.(type) {
+		case *osm.Way:
+			nodes := make([]int64, 0, len(o.Nodes))
+			for _, v := range o.Nodes {
+				nodes = append(nodes, int64(v.ID))
+			}
+			w := Way{
+				OsmID:     int64(o.ID),
+				Tags:      convertTags(o.Tags),
+				Nodes:     nodes,
+				Timestamp: o.Timestamp,
+				Version:   o.Version,
+				Visible:   o.Visible,
+			}
+			if _, err = ways.ReplaceOne(context.Background(), bson.M{"osm_id": int64(o.ID)}, w, opts); err != nil {
+				return err
+			}
+			wc++
+		case *osm.Node:
+			n := Node{
+				OsmID: int64(o.ID),
+				Location: Coords{
+					Type: "Point",
+					Coordinates: []float64{
+						o.Lon,
+						o.Lat,
+					}},
+				Tags:      convertTags(o.Tags),
+				Version:   o.Version,
+				Timestamp: o.Timestamp,
+				Visible:   o.Visible,
+			}
+			if _, err = nodes.ReplaceOne(context.Background(), bson.M{"osm_id": int64(o.ID)}, n, opts); err != nil {
+				return err
+			}
+			nc++
+		case *osm.Relation:
+			members := make([]Member, len(o.Members))
+			for _, v := range o.Members {
+				members = append(members, Member{
+					Type:        v.Type,
+					Version:     v.Version,
+					Orientation: v.Orientation,
+					Ref:         v.Ref,
+					Role:        v.Role,
+					Location: Coords{
+						Type: "Point",
+						Coordinates: []float64{
+							v.Lon,
+							v.Lat,
+						}},
+				})
+			}
+			r := Relation{
+				OsmID:     int64(o.ID),
+				Tags:      convertTags(o.Tags),
+				Version:   o.Version,
+				Timestamp: o.Timestamp,
+				Visible:   o.Visible,
+				Members:   members,
+			}
+			if _, err = relations.ReplaceOne(context.Background(), bson.M{"osm_id": int64(o.ID)}, r, opts); err != nil {
+				return err
+			}
+			rc++
+		}
+		fmt.Printf("\rNodes: %d Ways: %d Relations: %d", nc, wc, rc)
+	}
+
+	scanErr := scanner.Err()
+	if scanErr != nil {
+		return scanErr
+	}
+	return nil
+}
+
+func convertTags(tags osm.Tags) map[string]string {
+	result := make(map[string]string, len(tags))
+	for _, t := range tags {
+		result[t.Key] = t.Value
+	}
+	return result
+}
diff --git a/models.go b/models.go
new file mode 100644
index 0000000..1c7d0ee
--- /dev/null
+++ b/models.go
@@ -0,0 +1,57 @@
+package main
+
+import (
+	"time"
+
+	"github.com/paulmach/orb"
+	"github.com/paulmach/osm"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type Coords struct {
+	Type        string    `bson:"type"`
+	Coordinates []float64 `bson:"coordinates"`
+}
+
+type Node struct {
+	ID        primitive.ObjectID `bson:"_id,omitempty"`
+	OsmID     int64              `bson:"osm_id"`
+	Visible   bool               `bson:"visible"`
+	Version   int                `bson:"version,omitempty"`
+	Timestamp time.Time          `bson:"timestamp"`
+	Tags      map[string]string  `bson:"tags,omitempty"`
+	Location  Coords             `bson:"location"`
+}
+
+type Way struct {
+	ID        primitive.ObjectID `bson:"_id,omitempty"`
+	OsmID     int64              `bson:"osm_id"`
+	Visible   bool               `bson:"visible"`
+	Version   int                `bson:"version"`
+	Timestamp time.Time          `bson:"timestamp"`
+	Nodes     []int64            `bson:"nodes"`
+	Tags      map[string]string  `bson:"tags"`
+}
+
+type Relation struct {
+	ID        primitive.ObjectID `bson:"_id,omitempty"`
+	OsmID     int64              `bson:"osm_id"`
+	Visible   bool               `bson:"visible"`
+	Version   int                `bson:"version"`
+	Timestamp time.Time          `bson:"timestamp"`
+	Members   []Member           `bson:"members"`
+	Tags      map[string]string  `bson:"tags"`
+}
+
+type Member struct {
+	Type osm.Type `bson:"type"`
+	Ref  int64    `bson:"ref"`
+	Role string   `bson:"role"`
+
+	Version  int
+	Location Coords `bson:"location"`
+
+	// Orientation is the direction of the way around a ring of a multipolygon.
+	// Only valid for multipolygon or boundary relations.
+	Orientation orb.Orientation `bson:"orienation,omitempty"`
+}