diff --git a/cmd/rpdata-dump/main.go b/cmd/rpdata-dump/main.go new file mode 100644 index 0000000..f140711 --- /dev/null +++ b/cmd/rpdata-dump/main.go @@ -0,0 +1,246 @@ +package main + +import ( + "archive/zip" + "context" + "encoding/json" + "flag" + "fmt" + "git.aiterp.net/rpdata/api/database" + "git.aiterp.net/rpdata/api/database/mongodb" + "git.aiterp.net/rpdata/api/internal/config" + "git.aiterp.net/rpdata/api/models" + "log" + "os" + "time" +) + +var flagDriver = flag.String("driver", "mongodb", "The database driver to use.") +var flagHost = flag.String("host", "127.0.0.1", "The host to connect to.") +var flagPort = flag.Int("port", 27017, "The port to connect on.") +var flagDb = flag.String("db", "rpdata", "The database name") +var flagUsername = flag.String("username", "", "") +var flagPassword = flag.String("password", "", "") +var flagMechanism = flag.String("mechanism", "", "") +var flagOutputFile = flag.String("outfile", "dump.zip", "The file to write to.") +var flagIncludeKeys = flag.Bool("include-keys", false, "Whether to include the keys.") + +func main() { + flag.Parse() + + trueValue := true + truePtr := &trueValue + + cfg := config.Database{ + Driver: *flagDriver, + Host: *flagHost, + Port: *flagPort, + Db: *flagDb, + Username: *flagUsername, + Password: *flagPassword, + Mechanism: *flagMechanism, + } + + mongodb.DisableFixes = true + + db, err := database.Init(cfg) + if err != nil { + log.Fatalln("Failed to open database:", err) + } + + file, err := os.OpenFile(*flagOutputFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) + if err != nil { + log.Fatalln("Failed to open output file:", err) + } + + zipWriter := zip.NewWriter(file) + defer func() { + err = zipWriter.Close() + if err != nil { + log.Fatalln("Failed to close output file:", err) + } + }() + + ctx, cancel := context.WithTimeout(context.Background(), time.Minute*30) + defer cancel() + + log.Println("Dumping characters...") + characters, err := db.Characters().List(ctx, models.CharacterFilter{}) + if err != nil { + log.Println("Failed to get characters:", err) + } + if len(characters) == 0 { + log.Println("No characters to dump.") + } + for i, character := range characters { + err := writeJsonFile(zipWriter, "character", i, character.ID, time.Now(), character) + if err != nil { + log.Println("Failed to write character", character.ID, ":", err) + continue + } + } + + log.Println("Dumping channels...") + channels, err := db.Channels().List(ctx, models.ChannelFilter{}) + if err != nil { + log.Println("Failed to get channels:", err) + } + if len(channels) == 0 { + log.Println("No channels to dump.") + } + for i, channel := range channels { + err := writeJsonFile(zipWriter, "channel", i, channel.Name, time.Now(), channel) + if err != nil { + log.Println("Failed to write channel", channel.Name, ":", err) + continue + } + } + + log.Println("Dumping changes...") + changes, err := db.Changes().List(ctx, models.ChangeFilter{}) + if err != nil { + log.Println("Failed to get changes:", err) + } + if len(changes) == 0 { + log.Println("No changes to dump.") + } + for i, change := range changes { + err := writeJsonFile(zipWriter, "change", i, change.ID, change.Date, change) + if err != nil { + log.Println("Failed to write change", change.ID, ":", err) + continue + } + } + + log.Println("Dumping stories...") + stories, err := db.Stories().List(ctx, models.StoryFilter{}) + if err != nil { + log.Println("Failed to get stories:", err) + } + unlistedStories, err := db.Stories().List(ctx, models.StoryFilter{Unlisted: truePtr}) + if err != nil { + log.Println("Failed to get unlisted stories:", err) + } else { + stories = append(stories, unlistedStories...) + } + if len(stories) == 0 { + log.Println("No stories to dump.") + } + for i, story := range stories { + err := writeJsonFile(zipWriter, "story", i, story.ID, story.CreatedDate, story) + if err != nil { + log.Println("Failed to write story", story.ID, ":", err) + continue + } + } + + log.Println("Dumping chapters...") + chapters, err := db.Chapters().List(ctx, models.ChapterFilter{}) + if err != nil { + log.Println("Failed to get chapters:", err) + } + if len(chapters) == 0 { + log.Println("No chapters to dump.") + } + for i, chapter := range chapters { + err := writeJsonFile(zipWriter, "chapter", i, chapter.ID, chapter.CreatedDate, chapter) + if err != nil { + log.Println("Failed to write chapter", chapter.ID, ":", err) + continue + } + } + + log.Println("Dumping comments...") + comments, err := db.Comments().List(ctx, models.CommentFilter{}) + if err != nil { + log.Println("Failed to get comments:", err) + } + if len(comments) == 0 { + log.Println("No comments to dump.") + } + for i, comment := range comments { + err := writeJsonFile(zipWriter, "comment", i, comment.ID, comment.CreatedDate, comment) + if err != nil { + log.Println("Failed to write comment", comment.ID, ":", err) + continue + } + } + + log.Println("Dumping posts...") + posts, err := db.Posts().List(ctx, models.PostFilter{}) + if err != nil { + log.Println("Failed to get posts:", err) + } + if len(posts) == 0 { + log.Println("No posts to dump.") + } + for i, post := range posts { + err := writeJsonFile(zipWriter, "post", i, post.ID, post.Time, post) + if err != nil { + log.Println("Failed to write post", post.ID, ":", err) + continue + } + } + + log.Println("Dumping logs...") + logs, err := db.Logs().List(ctx, models.LogFilter{}) + if err != nil { + log.Println("Failed to get logs:", err) + } + if len(logs) == 0 { + log.Println("No logs to dump.") + } + for i, logEntry := range logs { + err := writeJsonFile(zipWriter, "post", i, logEntry.ID, logEntry.Date, logEntry) + if err != nil { + log.Println("Failed to write post", logEntry.ID, ":", err) + continue + } + } + + log.Println("Dumping users...") + users, err := db.Users().List(ctx) + if err != nil { + log.Println("Failed to get users:", err) + } + if len(users) == 0 { + log.Println("No users to dump.") + } + for i, userEntry := range users { + err := writeJsonFile(zipWriter, "user", i, userEntry.ID, time.Now(), userEntry) + if err != nil { + log.Println("Failed to write user", userEntry.ID, ":", err) + continue + } + } + + if *flagIncludeKeys { + log.Println("Dumping keys...") + keys, err := db.Keys().List(ctx, models.KeyFilter{}) + if err != nil { + log.Println("Failed to get users:", err) + } + if len(keys) == 0 { + log.Println("No users to dump.") + } + for i, keyEntry := range keys { + err := writeJsonFile(zipWriter, "key", i, keyEntry.ID, time.Now(), keyEntry) + if err != nil { + log.Println("Failed to write key", keyEntry.ID, ":", err) + continue + } + } + } +} + +func writeJsonFile(zw *zip.Writer, model string, idx int, id string, t time.Time, data interface{}) error { + w, err := zw.CreateHeader(&zip.FileHeader{ + Name: fmt.Sprintf("rpdata_dump_v1/%s/%06d_%s.json", model, idx, id), + Modified: t, + }) + if err != nil { + return err + } + + return json.NewEncoder(w).Encode(data) +} diff --git a/database/mongodb/db.go b/database/mongodb/db.go index d8ecd38..837c7a4 100644 --- a/database/mongodb/db.go +++ b/database/mongodb/db.go @@ -11,6 +11,8 @@ import ( "github.com/globalsign/mgo" ) +var DisableFixes bool + type MongoDB struct { session *mgo.Session @@ -169,7 +171,9 @@ func Init(cfg config.Database) (*MongoDB, error) { return nil, err } - go posts.fixPositions(logs) + if !DisableFixes { + go posts.fixPositions(logs) + } return &MongoDB{ session: session, diff --git a/database/mongodb/posts.go b/database/mongodb/posts.go index 5556231..1c8c144 100644 --- a/database/mongodb/posts.go +++ b/database/mongodb/posts.go @@ -91,7 +91,12 @@ func (r *postRepository) List(ctx context.Context, filter models.PostFilter) ([] } posts := make([]*models.Post, 0, 32) - err := r.posts.Find(query).Sort("-logId", "position").Limit(filter.Limit).All(&posts) + var err error + if filter.LogID != nil { + err = r.posts.Find(query).Sort("position").Limit(filter.Limit).All(&posts) + } else { + err = r.posts.Find(query).Limit(filter.Limit).All(&posts) + } if err != nil { if err == mgo.ErrNotFound { return []*models.Post{}, nil diff --git a/database/mongodb/users.go b/database/mongodb/users.go index 01e3891..f2043af 100644 --- a/database/mongodb/users.go +++ b/database/mongodb/users.go @@ -5,6 +5,7 @@ import ( "git.aiterp.net/rpdata/api/models" "git.aiterp.net/rpdata/api/repositories" "github.com/globalsign/mgo" + "github.com/globalsign/mgo/bson" ) type userRepository struct { @@ -31,6 +32,16 @@ func (r *userRepository) Find(ctx context.Context, id string) (*models.User, err return user, nil } +func (r *userRepository) List(ctx context.Context) ([]*models.User, error) { + users := make([]*models.User, 0, 8) + err := r.users.Find(bson.M{}).All(&users) + if err != nil && err != mgo.ErrNotFound { + return nil, err + } + + return users, nil +} + func (r *userRepository) Insert(ctx context.Context, user models.User) (*models.User, error) { err := r.users.Insert(user) if err != nil { diff --git a/go.mod b/go.mod index 894ab7a..48a2b2e 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/globalsign/mgo v0.0.0-20180403085842-f76e4f9da92e github.com/go-ini/ini v1.35.0 // indirect github.com/golang/protobuf v1.3.1 // indirect + github.com/golang/snappy v0.0.2 // indirect github.com/google/go-cmp v0.3.0 // indirect github.com/gorilla/websocket v1.4.0 // indirect github.com/h2non/filetype v1.0.8 @@ -23,6 +24,8 @@ require ( github.com/tidwall/pretty v1.0.0 // indirect github.com/vektah/dataloaden v0.3.0 github.com/vektah/gqlparser v1.1.2 + github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c // indirect + github.com/xdg/stringprep v1.0.0 // indirect go.mongodb.org/mongo-driver v1.0.3 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 // indirect golang.org/x/net v0.0.0-20190514140710-3ec191127204 // indirect diff --git a/go.sum b/go.sum index aef6733..343f61b 100644 --- a/go.sum +++ b/go.sum @@ -29,6 +29,8 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.2 h1:aeE13tS0IiQgFjYdoL8qN3K1N2bXXtI6Vi51/y7BpMw= +github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= @@ -104,6 +106,10 @@ github.com/vektah/dataloaden v0.3.0 h1:ZfVN2QD6swgvp+tDqdH/OIT/wu3Dhu0cus0k5gIZS github.com/vektah/dataloaden v0.3.0/go.mod h1:/HUdMve7rvxZma+2ZELQeNh88+003LL7Pf/CZ089j8U= github.com/vektah/gqlparser v1.1.2 h1:ZsyLGn7/7jDNI+y4SEhI4yAxRChlv15pUHMjijT+e68= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0= +github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= go.mongodb.org/mongo-driver v1.0.3 h1:GKoji1ld3tw2aC+GX1wbr/J2fX13yNacEYoJ8Nhr0yU= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I= diff --git a/repositories/user.go b/repositories/user.go index f930d2d..49d51a6 100644 --- a/repositories/user.go +++ b/repositories/user.go @@ -7,5 +7,6 @@ import ( type UserRepository interface { Find(ctx context.Context, id string) (*models.User, error) + List(ctx context.Context) ([]*models.User, error) Insert(ctx context.Context, user models.User) (*models.User, error) }