add rpdata-dump command.

cmd/rpdata-dump/main.go

@@ -0,0 +1,246 @@
+package main
+
+import (
+	"archive/zip"
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"git.aiterp.net/rpdata/api/database"
+	"git.aiterp.net/rpdata/api/database/mongodb"
+	"git.aiterp.net/rpdata/api/internal/config"
+	"git.aiterp.net/rpdata/api/models"
+	"log"
+	"os"
+	"time"
+)
+
+var flagDriver = flag.String("driver", "mongodb", "The database driver to use.")
+var flagHost = flag.String("host", "127.0.0.1", "The host to connect to.")
+var flagPort = flag.Int("port", 27017, "The port to connect on.")
+var flagDb = flag.String("db", "rpdata", "The database name")
+var flagUsername = flag.String("username", "", "")
+var flagPassword = flag.String("password", "", "")
+var flagMechanism = flag.String("mechanism", "", "")
+var flagOutputFile = flag.String("outfile", "dump.zip", "The file to write to.")
+var flagIncludeKeys = flag.Bool("include-keys", false, "Whether to include the keys.")
+
+func main() {
+	flag.Parse()
+
+	trueValue := true
+	truePtr := &trueValue
+
+	cfg := config.Database{
+		Driver:    *flagDriver,
+		Host:      *flagHost,
+		Port:      *flagPort,
+		Db:        *flagDb,
+		Username:  *flagUsername,
+		Password:  *flagPassword,
+		Mechanism: *flagMechanism,
+	}
+
+	mongodb.DisableFixes = true
+
+	db, err := database.Init(cfg)
+	if err != nil {
+		log.Fatalln("Failed to open database:", err)
+	}
+
+	file, err := os.OpenFile(*flagOutputFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
+	if err != nil {
+		log.Fatalln("Failed to open output file:", err)
+	}
+
+	zipWriter := zip.NewWriter(file)
+	defer func() {
+		err = zipWriter.Close()
+		if err != nil {
+			log.Fatalln("Failed to close output file:", err)
+		}
+	}()
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Minute*30)
+	defer cancel()
+
+	log.Println("Dumping characters...")
+	characters, err := db.Characters().List(ctx, models.CharacterFilter{})
+	if err != nil {
+		log.Println("Failed to get characters:", err)
+	}
+	if len(characters) == 0 {
+		log.Println("No characters to dump.")
+	}
+	for i, character := range characters {
+		err := writeJsonFile(zipWriter, "character", i, character.ID, time.Now(), character)
+		if err != nil {
+			log.Println("Failed to write character", character.ID, ":", err)
+			continue
+		}
+	}
+
+	log.Println("Dumping channels...")
+	channels, err := db.Channels().List(ctx, models.ChannelFilter{})
+	if err != nil {
+		log.Println("Failed to get channels:", err)
+	}
+	if len(channels) == 0 {
+		log.Println("No channels to dump.")
+	}
+	for i, channel := range channels {
+		err := writeJsonFile(zipWriter, "channel", i, channel.Name, time.Now(), channel)
+		if err != nil {
+			log.Println("Failed to write channel", channel.Name, ":", err)
+			continue
+		}
+	}
+
+	log.Println("Dumping changes...")
+	changes, err := db.Changes().List(ctx, models.ChangeFilter{})
+	if err != nil {
+		log.Println("Failed to get changes:", err)
+	}
+	if len(changes) == 0 {
+		log.Println("No changes to dump.")
+	}
+	for i, change := range changes {
+		err := writeJsonFile(zipWriter, "change", i, change.ID, change.Date, change)
+		if err != nil {
+			log.Println("Failed to write change", change.ID, ":", err)
+			continue
+		}
+	}
+
+	log.Println("Dumping stories...")
+	stories, err := db.Stories().List(ctx, models.StoryFilter{})
+	if err != nil {
+		log.Println("Failed to get stories:", err)
+	}
+	unlistedStories, err := db.Stories().List(ctx, models.StoryFilter{Unlisted: truePtr})
+	if err != nil {
+		log.Println("Failed to get unlisted stories:", err)
+	} else {
+		stories = append(stories, unlistedStories...)
+	}
+	if len(stories) == 0 {
+		log.Println("No stories to dump.")
+	}
+	for i, story := range stories {
+		err := writeJsonFile(zipWriter, "story", i, story.ID, story.CreatedDate, story)
+		if err != nil {
+			log.Println("Failed to write story", story.ID, ":", err)
+			continue
+		}
+	}
+
+	log.Println("Dumping chapters...")
+	chapters, err := db.Chapters().List(ctx, models.ChapterFilter{})
+	if err != nil {
+		log.Println("Failed to get chapters:", err)
+	}
+	if len(chapters) == 0 {
+		log.Println("No chapters to dump.")
+	}
+	for i, chapter := range chapters {
+		err := writeJsonFile(zipWriter, "chapter", i, chapter.ID, chapter.CreatedDate, chapter)
+		if err != nil {
+			log.Println("Failed to write chapter", chapter.ID, ":", err)
+			continue
+		}
+	}
+
+	log.Println("Dumping comments...")
+	comments, err := db.Comments().List(ctx, models.CommentFilter{})
+	if err != nil {
+		log.Println("Failed to get comments:", err)
+	}
+	if len(comments) == 0 {
+		log.Println("No comments to dump.")
+	}
+	for i, comment := range comments {
+		err := writeJsonFile(zipWriter, "comment", i, comment.ID, comment.CreatedDate, comment)
+		if err != nil {
+			log.Println("Failed to write comment", comment.ID, ":", err)
+			continue
+		}
+	}
+
+	log.Println("Dumping posts...")
+	posts, err := db.Posts().List(ctx, models.PostFilter{})
+	if err != nil {
+		log.Println("Failed to get posts:", err)
+	}
+	if len(posts) == 0 {
+		log.Println("No posts to dump.")
+	}
+	for i, post := range posts {
+		err := writeJsonFile(zipWriter, "post", i, post.ID, post.Time, post)
+		if err != nil {
+			log.Println("Failed to write post", post.ID, ":", err)
+			continue
+		}
+	}
+
+	log.Println("Dumping logs...")
+	logs, err := db.Logs().List(ctx, models.LogFilter{})
+	if err != nil {
+		log.Println("Failed to get logs:", err)
+	}
+	if len(logs) == 0 {
+		log.Println("No logs to dump.")
+	}
+	for i, logEntry := range logs {
+		err := writeJsonFile(zipWriter, "post", i, logEntry.ID, logEntry.Date, logEntry)
+		if err != nil {
+			log.Println("Failed to write post", logEntry.ID, ":", err)
+			continue
+		}
+	}
+
+	log.Println("Dumping users...")
+	users, err := db.Users().List(ctx)
+	if err != nil {
+		log.Println("Failed to get users:", err)
+	}
+	if len(users) == 0 {
+		log.Println("No users to dump.")
+	}
+	for i, userEntry := range users {
+		err := writeJsonFile(zipWriter, "user", i, userEntry.ID, time.Now(), userEntry)
+		if err != nil {
+			log.Println("Failed to write user", userEntry.ID, ":", err)
+			continue
+		}
+	}
+
+	if *flagIncludeKeys {
+		log.Println("Dumping keys...")
+		keys, err := db.Keys().List(ctx, models.KeyFilter{})
+		if err != nil {
+			log.Println("Failed to get users:", err)
+		}
+		if len(keys) == 0 {
+			log.Println("No users to dump.")
+		}
+		for i, keyEntry := range keys {
+			err := writeJsonFile(zipWriter, "key", i, keyEntry.ID, time.Now(), keyEntry)
+			if err != nil {
+				log.Println("Failed to write key", keyEntry.ID, ":", err)
+				continue
+			}
+		}
+	}
+}
+
+func writeJsonFile(zw *zip.Writer, model string, idx int, id string, t time.Time, data interface{}) error {
+	w, err := zw.CreateHeader(&zip.FileHeader{
+		Name:     fmt.Sprintf("rpdata_dump_v1/%s/%06d_%s.json", model, idx, id),
+		Modified: t,
+	})
+	if err != nil {
+		return err
+	}
+
+	return json.NewEncoder(w).Encode(data)
+}

database/mongodb/db.go

@@ -11,6 +11,8 @@ import (
 	"github.com/globalsign/mgo"
 )
 
+var DisableFixes bool
+
 type MongoDB struct {
 	session *mgo.Session
 
@@ -169,7 +171,9 @@ func Init(cfg config.Database) (*MongoDB, error) {
 		return nil, err
 	}
 
-	go posts.fixPositions(logs)
+	if !DisableFixes {
+		go posts.fixPositions(logs)
+	}
 
 	return &MongoDB{
 		session: session,

database/mongodb/posts.go

@@ -91,7 +91,12 @@ func (r *postRepository) List(ctx context.Context, filter models.PostFilter) ([]
 	}
 
 	posts := make([]*models.Post, 0, 32)
-	err := r.posts.Find(query).Sort("-logId", "position").Limit(filter.Limit).All(&posts)
+	var err error
+	if filter.LogID != nil {
+		err = r.posts.Find(query).Sort("position").Limit(filter.Limit).All(&posts)
+	} else {
+		err = r.posts.Find(query).Limit(filter.Limit).All(&posts)
+	}
 	if err != nil {
 		if err == mgo.ErrNotFound {
 			return []*models.Post{}, nil

database/mongodb/users.go

@@ -5,6 +5,7 @@ import (
 	"git.aiterp.net/rpdata/api/models"
 	"git.aiterp.net/rpdata/api/repositories"
 	"github.com/globalsign/mgo"
+	"github.com/globalsign/mgo/bson"
 )
 
 type userRepository struct {
@@ -31,6 +32,16 @@ func (r *userRepository) Find(ctx context.Context, id string) (*models.User, err
 	return user, nil
 }
 
+func (r *userRepository) List(ctx context.Context) ([]*models.User, error) {
+	users := make([]*models.User, 0, 8)
+	err := r.users.Find(bson.M{}).All(&users)
+	if err != nil && err != mgo.ErrNotFound {
+		return nil, err
+	}
+
+	return users, nil
+}
+
 func (r *userRepository) Insert(ctx context.Context, user models.User) (*models.User, error) {
 	err := r.users.Insert(user)
 	if err != nil {

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/globalsign/mgo v0.0.0-20180403085842-f76e4f9da92e
 	github.com/go-ini/ini v1.35.0 // indirect
 	github.com/golang/protobuf v1.3.1 // indirect
+	github.com/golang/snappy v0.0.2 // indirect
 	github.com/google/go-cmp v0.3.0 // indirect
 	github.com/gorilla/websocket v1.4.0 // indirect
 	github.com/h2non/filetype v1.0.8
@@ -23,6 +24,8 @@ require (
 	github.com/tidwall/pretty v1.0.0 // indirect
 	github.com/vektah/dataloaden v0.3.0
 	github.com/vektah/gqlparser v1.1.2
+	github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c // indirect
+	github.com/xdg/stringprep v1.0.0 // indirect
 	go.mongodb.org/mongo-driver v1.0.3
 	golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 // indirect
 	golang.org/x/net v0.0.0-20190514140710-3ec191127204 // indirect

go.sum

@@ -29,6 +29,8 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/snappy v0.0.2 h1:aeE13tS0IiQgFjYdoL8qN3K1N2bXXtI6Vi51/y7BpMw=
+github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
@@ -104,6 +106,10 @@ github.com/vektah/dataloaden v0.3.0 h1:ZfVN2QD6swgvp+tDqdH/OIT/wu3Dhu0cus0k5gIZS
 github.com/vektah/dataloaden v0.3.0/go.mod h1:/HUdMve7rvxZma+2ZELQeNh88+003LL7Pf/CZ089j8U=
 github.com/vektah/gqlparser v1.1.2 h1:ZsyLGn7/7jDNI+y4SEhI4yAxRChlv15pUHMjijT+e68=
 github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
+github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk=
+github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
+github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0=
+github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
 go.mongodb.org/mongo-driver v1.0.3 h1:GKoji1ld3tw2aC+GX1wbr/J2fX13yNacEYoJ8Nhr0yU=
 go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I=

repositories/user.go

@@ -7,5 +7,6 @@ import (
 
 type UserRepository interface {
 	Find(ctx context.Context, id string) (*models.User, error)
+	List(ctx context.Context) ([]*models.User, error)
 	Insert(ctx context.Context, user models.User) (*models.User, error)
 }