aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.gitignore2
-rw-r--r--README.md22
-rw-r--r--go.mod20
-rw-r--r--go.sum42
-rw-r--r--main.go191
-rw-r--r--models.go57
6 files changed, 334 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..2a550d9
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+.idea
+osm2mgo \ No newline at end of file
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..c52dd76
--- /dev/null
+++ b/README.md
@@ -0,0 +1,22 @@
+# OpenStreetMaps to Mongo
+
+Simple loader from osm dump file to mongodb. Based on https://github.com/paulmach/osm package.
+
+## Build
+
+`go build -o osm2go`
+
+## Usage
+
+`./osm2go -osmfile PATH_TO_OSM_FILE [-dbconnection mongodb://localhost:27017] [-dbname osm]`
+
+* `osmfile` required, path to *.osm or *.osm.pbf file
+* `dbconnection` optional, mongodb connection string (default: `mongodb://localhost:27017`)
+* `dbname` optional, mongodb database name (default: `osm`)
+
+## Example
+
+```
+# ./osm2mgo -osmfile ~/Downloads/RU.pbf
+Nodes: 1294069 Ways: 0 Relations: 0
+``` \ No newline at end of file
diff --git a/go.mod b/go.mod
new file mode 100644
index 0000000..06c3118
--- /dev/null
+++ b/go.mod
@@ -0,0 +1,20 @@
+module osm2mongo
+
+go 1.12
+
+require (
+ github.com/go-stack/stack v1.8.0 // indirect
+ github.com/gogo/protobuf v1.2.1 // indirect
+ github.com/golang/snappy v0.0.1 // indirect
+ github.com/google/go-cmp v0.3.0 // indirect
+ github.com/paulmach/orb v0.1.3
+ github.com/paulmach/osm v0.0.1
+ github.com/stretchr/testify v1.3.0 // indirect
+ github.com/tidwall/pretty v0.0.0-20190325153808-1166b9ac2b65 // indirect
+ github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c // indirect
+ github.com/xdg/stringprep v1.0.0 // indirect
+ go.mongodb.org/mongo-driver v1.0.2
+ golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f // indirect
+ golang.org/x/sync v0.0.0-20190423024810-112230192c58 // indirect
+ golang.org/x/text v0.3.2 // indirect
+)
diff --git a/go.sum b/go.sum
new file mode 100644
index 0000000..419da2d
--- /dev/null
+++ b/go.sum
@@ -0,0 +1,42 @@
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
+github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/paulmach/orb v0.1.3 h1:Wa1nzU269Zv7V9paVEY1COWW8FCqv4PC/KJRbJSimpM=
+github.com/paulmach/orb v0.1.3/go.mod h1:VFlX/8C+IQ1p6FTRRKzKoOPJnvEtA5G0Veuqwbu//Vk=
+github.com/paulmach/osm v0.0.1 h1:TxU/uZnJ+ssntblY6kSieOP4tQv31VW8wfyf4L3BeKs=
+github.com/paulmach/osm v0.0.1/go.mod h1:d6Ez0X1es6TaiYBxNmn1WbutRRQleZdvpoqeWdH/5kM=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/tidwall/pretty v0.0.0-20190325153808-1166b9ac2b65 h1:rQ229MBgvW68s1/g6f1/63TgYwYxfF4E+bi/KC19P8g=
+github.com/tidwall/pretty v0.0.0-20190325153808-1166b9ac2b65/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk=
+github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
+github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0=
+github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
+go.mongodb.org/mongo-driver v1.0.2 h1:RwjK1tKt7VPqQh3tsjiEqKJg75GNhP/loch+PwRc4ig=
+go.mongodb.org/mongo-driver v1.0.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f h1:R423Cnkcp5JABoeemiGEPlt9tHXFfw5kvc0yqlxRPWo=
+golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
diff --git a/main.go b/main.go
new file mode 100644
index 0000000..1458fe8
--- /dev/null
+++ b/main.go
@@ -0,0 +1,191 @@
+package main
+
+import (
+ "context"
+ "flag"
+ "fmt"
+ "log"
+ "os"
+ "time"
+
+ "github.com/paulmach/osm"
+ "github.com/paulmach/osm/osmpbf"
+ "go.mongodb.org/mongo-driver/bson"
+ "go.mongodb.org/mongo-driver/mongo"
+ "go.mongodb.org/mongo-driver/mongo/options"
+ "go.mongodb.org/mongo-driver/x/bsonx"
+)
+
+func main() {
+ dbconnection := flag.String("dbconnection", "mongodb://localhost:27017", "Mongo database name")
+ dbname := flag.String("dbname", "osm", "Mongo database name")
+ osmfile := flag.String("osmfile", "", "OSM file")
+ flag.Parse()
+ ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
+ client, err := mongo.Connect(ctx, options.Client().ApplyURI(*dbconnection))
+ if err != nil {
+ log.Fatal(err)
+ }
+ defer client.Disconnect(context.Background())
+ db := client.Database(*dbname)
+ if err := read(db, *osmfile); err != nil {
+ log.Fatal(err)
+ }
+
+}
+
+func read(db *mongo.Database, file string) error {
+ nodes := db.Collection("nodes")
+ _, _ = nodes.Indexes().CreateOne(
+ context.Background(),
+ mongo.IndexModel{
+ Keys: bsonx.Doc{{"osm_id", bsonx.Int32(1)}},
+ Options: options.Index().SetUnique(true).SetSparse(true),
+ },
+ )
+ _, _ = nodes.Indexes().CreateOne(
+ context.Background(),
+ mongo.IndexModel{
+ Keys: bsonx.Doc{{"coords", bsonx.Int32(1)}},
+ Options: options.Index().SetSphereVersion(2).SetSparse(true),
+ },
+ )
+
+ ways := db.Collection("ways")
+ _, _ = ways.Indexes().CreateOne(
+ context.Background(),
+ mongo.IndexModel{
+ Keys: bsonx.Doc{{"osm_id", bsonx.Int32(1)}},
+ Options: options.Index().SetUnique(true).SetSparse(true),
+ },
+ )
+ _, _ = ways.Indexes().CreateOne(
+ context.Background(),
+ mongo.IndexModel{
+ Keys: bsonx.Doc{{"nodes", bsonx.Int32(1)}},
+ Options: options.Index().SetSparse(true),
+ },
+ )
+
+ relations := db.Collection("relations")
+ _, _ = nodes.Indexes().CreateOne(
+ context.Background(),
+ mongo.IndexModel{
+ Keys: bsonx.Doc{{"osm_id", bsonx.Int32(1)}},
+ Options: options.Index().SetUnique(true).SetSparse(true),
+ },
+ )
+ _, _ = nodes.Indexes().CreateOne(
+ context.Background(),
+ mongo.IndexModel{
+ Keys: bsonx.Doc{{"members.ref", bsonx.Int32(1)}},
+ Options: options.Index().SetUnique(true).SetSparse(true),
+ },
+ )
+ _, _ = nodes.Indexes().CreateOne(
+ context.Background(),
+ mongo.IndexModel{
+ Keys: bsonx.Doc{{"members.coords", bsonx.Int32(1)}},
+ Options: options.Index().SetSphereVersion(2).SetSparse(true),
+ },
+ )
+
+ f, err := os.Open(file)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+
+ opts := (new(options.ReplaceOptions)).SetUpsert(true)
+ nc := 0
+ wc := 0
+ rc := 0
+
+ scanner := osmpbf.New(context.Background(), f, 3)
+ defer scanner.Close()
+
+ for scanner.Scan() {
+ o := scanner.Object()
+ switch o := o.(type) {
+ case *osm.Way:
+ nodes := make([]int64, 0, len(o.Nodes))
+ for _, v := range o.Nodes {
+ nodes = append(nodes, int64(v.ID))
+ }
+ w := Way{
+ OsmID: int64(o.ID),
+ Tags: convertTags(o.Tags),
+ Nodes: nodes,
+ Timestamp: o.Timestamp,
+ Version: o.Version,
+ Visible: o.Visible,
+ }
+ if _, err = ways.ReplaceOne(context.Background(), bson.M{"osm_id": int64(o.ID)}, w, opts); err != nil {
+ return err
+ }
+ wc++
+ case *osm.Node:
+ n := Node{
+ OsmID: int64(o.ID),
+ Location: Coords{
+ Type: "Point",
+ Coordinates: []float64{
+ o.Lon,
+ o.Lat,
+ }},
+ Tags: convertTags(o.Tags),
+ Version: o.Version,
+ Timestamp: o.Timestamp,
+ Visible: o.Visible,
+ }
+ if _, err = nodes.ReplaceOne(context.Background(), bson.M{"osm_id": int64(o.ID)}, n, opts); err != nil {
+ return err
+ }
+ nc++
+ case *osm.Relation:
+ members := make([]Member, len(o.Members))
+ for _, v := range o.Members {
+ members = append(members, Member{
+ Type: v.Type,
+ Version: v.Version,
+ Orientation: v.Orientation,
+ Ref: v.Ref,
+ Role: v.Role,
+ Location: Coords{
+ Type: "Point",
+ Coordinates: []float64{
+ v.Lon,
+ v.Lat,
+ }},
+ })
+ }
+ r := Relation{
+ OsmID: int64(o.ID),
+ Tags: convertTags(o.Tags),
+ Version: o.Version,
+ Timestamp: o.Timestamp,
+ Visible: o.Visible,
+ Members: members,
+ }
+ if _, err = relations.ReplaceOne(context.Background(), bson.M{"osm_id": int64(o.ID)}, r, opts); err != nil {
+ return err
+ }
+ rc++
+ }
+ fmt.Printf("\rNodes: %d Ways: %d Relations: %d", nc, wc, rc)
+ }
+
+ scanErr := scanner.Err()
+ if scanErr != nil {
+ return scanErr
+ }
+ return nil
+}
+
+func convertTags(tags osm.Tags) map[string]string {
+ result := make(map[string]string, len(tags))
+ for _, t := range tags {
+ result[t.Key] = t.Value
+ }
+ return result
+}
diff --git a/models.go b/models.go
new file mode 100644
index 0000000..1c7d0ee
--- /dev/null
+++ b/models.go
@@ -0,0 +1,57 @@
+package main
+
+import (
+ "time"
+
+ "github.com/paulmach/orb"
+ "github.com/paulmach/osm"
+ "go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type Coords struct {
+ Type string `bson:"type"`
+ Coordinates []float64 `bson:"coordinates"`
+}
+
+type Node struct {
+ ID primitive.ObjectID `bson:"_id,omitempty"`
+ OsmID int64 `bson:"osm_id"`
+ Visible bool `bson:"visible"`
+ Version int `bson:"version,omitempty"`
+ Timestamp time.Time `bson:"timestamp"`
+ Tags map[string]string `bson:"tags,omitempty"`
+ Location Coords `bson:"location"`
+}
+
+type Way struct {
+ ID primitive.ObjectID `bson:"_id,omitempty"`
+ OsmID int64 `bson:"osm_id"`
+ Visible bool `bson:"visible"`
+ Version int `bson:"version"`
+ Timestamp time.Time `bson:"timestamp"`
+ Nodes []int64 `bson:"nodes"`
+ Tags map[string]string `bson:"tags"`
+}
+
+type Relation struct {
+ ID primitive.ObjectID `bson:"_id,omitempty"`
+ OsmID int64 `bson:"osm_id"`
+ Visible bool `bson:"visible"`
+ Version int `bson:"version"`
+ Timestamp time.Time `bson:"timestamp"`
+ Members []Member `bson:"members"`
+ Tags map[string]string `bson:"tags"`
+}
+
+type Member struct {
+ Type osm.Type `bson:"type"`
+ Ref int64 `bson:"ref"`
+ Role string `bson:"role"`
+
+ Version int
+ Location Coords `bson:"location"`
+
+ // Orientation is the direction of the way around a ring of a multipolygon.
+ // Only valid for multipolygon or boundary relations.
+ Orientation orb.Orientation `bson:"orienation,omitempty"`
+}
generated by cgit v1.2.3 (git 2.39.1) at 2025-01-18 10:44:13 +0300