rpdata/api
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

add rpdata-dump command.

Browse Source
This commit is contained in:
gisle
2021-03-20 14:24:02 +01:00
parent 570d0e2bd2
commit c9084287a4
7 changed files with 278 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/rpdata-dump/main.go
+246
View File
@@ -0,0 +1,246 @@
package main
import (
"archive/zip"
"context"
"encoding/json"
"flag"
"fmt"
"git.aiterp.net/rpdata/api/database"
"git.aiterp.net/rpdata/api/database/mongodb"
"git.aiterp.net/rpdata/api/internal/config"
"git.aiterp.net/rpdata/api/models"
"log"
"os"
"time"
)
var flagDriver = flag.String("driver", "mongodb", "The database driver to use.")
var flagHost = flag.String("host", "127.0.0.1", "The host to connect to.")
var flagPort = flag.Int("port", 27017, "The port to connect on.")
var flagDb = flag.String("db", "rpdata", "The database name")
var flagUsername = flag.String("username", "", "")
var flagPassword = flag.String("password", "", "")
var flagMechanism = flag.String("mechanism", "", "")
var flagOutputFile = flag.String("outfile", "dump.zip", "The file to write to.")
var flagIncludeKeys = flag.Bool("include-keys", false, "Whether to include the keys.")
func main() {
flag.Parse()
trueValue := true
truePtr := &trueValue
cfg := config.Database{
Driver: *flagDriver,
Host: *flagHost,
Port: *flagPort,
Db: *flagDb,
Username: *flagUsername,
Password: *flagPassword,
Mechanism: *flagMechanism,
}
mongodb.DisableFixes = true
db, err := database.Init(cfg)
if err != nil {
log.Fatalln("Failed to open database:", err)
}
file, err := os.OpenFile(*flagOutputFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
if err != nil {
log.Fatalln("Failed to open output file:", err)
}
zipWriter := zip.NewWriter(file)
defer func() {
err = zipWriter.Close()
if err != nil {
log.Fatalln("Failed to close output file:", err)
}
}()
ctx, cancel := context.WithTimeout(context.Background(), time.Minute*30)
defer cancel()
log.Println("Dumping characters...")
characters, err := db.Characters().List(ctx, models.CharacterFilter{})
if err != nil {
log.Println("Failed to get characters:", err)
}
if len(characters) == 0 {
log.Println("No characters to dump.")
}
for i, character := range characters {
err := writeJsonFile(zipWriter, "character", i, character.ID, time.Now(), character)
if err != nil {
log.Println("Failed to write character", character.ID, ":", err)
continue
}
}
log.Println("Dumping channels...")
channels, err := db.Channels().List(ctx, models.ChannelFilter{})
if err != nil {
log.Println("Failed to get channels:", err)
}
if len(channels) == 0 {
log.Println("No channels to dump.")
}
for i, channel := range channels {
err := writeJsonFile(zipWriter, "channel", i, channel.Name, time.Now(), channel)
if err != nil {
log.Println("Failed to write channel", channel.Name, ":", err)
continue
}
}
log.Println("Dumping changes...")
changes, err := db.Changes().List(ctx, models.ChangeFilter{})
if err != nil {
log.Println("Failed to get changes:", err)
}
if len(changes) == 0 {
log.Println("No changes to dump.")
}
for i, change := range changes {
err := writeJsonFile(zipWriter, "change", i, change.ID, change.Date, change)
if err != nil {
log.Println("Failed to write change", change.ID, ":", err)
continue
}
}
log.Println("Dumping stories...")
stories, err := db.Stories().List(ctx, models.StoryFilter{})
if err != nil {
log.Println("Failed to get stories:", err)
}
unlistedStories, err := db.Stories().List(ctx, models.StoryFilter{Unlisted: truePtr})
if err != nil {
log.Println("Failed to get unlisted stories:", err)
} else {
stories = append(stories, unlistedStories...)
}
if len(stories) == 0 {
log.Println("No stories to dump.")
}
for i, story := range stories {
err := writeJsonFile(zipWriter, "story", i, story.ID, story.CreatedDate, story)
if err != nil {
log.Println("Failed to write story", story.ID, ":", err)
continue
}
}
log.Println("Dumping chapters...")
chapters, err := db.Chapters().List(ctx, models.ChapterFilter{})
if err != nil {
log.Println("Failed to get chapters:", err)
}
if len(chapters) == 0 {
log.Println("No chapters to dump.")
}
for i, chapter := range chapters {
err := writeJsonFile(zipWriter, "chapter", i, chapter.ID, chapter.CreatedDate, chapter)
if err != nil {
log.Println("Failed to write chapter", chapter.ID, ":", err)
continue
}
}
log.Println("Dumping comments...")
comments, err := db.Comments().List(ctx, models.CommentFilter{})
if err != nil {
log.Println("Failed to get comments:", err)
}
if len(comments) == 0 {
log.Println("No comments to dump.")
}
for i, comment := range comments {
err := writeJsonFile(zipWriter, "comment", i, comment.ID, comment.CreatedDate, comment)
if err != nil {
log.Println("Failed to write comment", comment.ID, ":", err)
continue
}
}
log.Println("Dumping posts...")
posts, err := db.Posts().List(ctx, models.PostFilter{})
if err != nil {
log.Println("Failed to get posts:", err)
}
if len(posts) == 0 {
log.Println("No posts to dump.")
}
for i, post := range posts {
err := writeJsonFile(zipWriter, "post", i, post.ID, post.Time, post)
if err != nil {
log.Println("Failed to write post", post.ID, ":", err)
continue
}
}
log.Println("Dumping logs...")
logs, err := db.Logs().List(ctx, models.LogFilter{})
if err != nil {
log.Println("Failed to get logs:", err)
}
if len(logs) == 0 {
log.Println("No logs to dump.")
}
for i, logEntry := range logs {
err := writeJsonFile(zipWriter, "post", i, logEntry.ID, logEntry.Date, logEntry)
if err != nil {
log.Println("Failed to write post", logEntry.ID, ":", err)
continue
}
}
log.Println("Dumping users...")
users, err := db.Users().List(ctx)
if err != nil {
log.Println("Failed to get users:", err)
}
if len(users) == 0 {
log.Println("No users to dump.")
}
for i, userEntry := range users {
err := writeJsonFile(zipWriter, "user", i, userEntry.ID, time.Now(), userEntry)
if err != nil {
log.Println("Failed to write user", userEntry.ID, ":", err)
continue
}
}
if *flagIncludeKeys {
log.Println("Dumping keys...")
keys, err := db.Keys().List(ctx, models.KeyFilter{})
if err != nil {
log.Println("Failed to get users:", err)
}
if len(keys) == 0 {
log.Println("No users to dump.")
}
for i, keyEntry := range keys {
err := writeJsonFile(zipWriter, "key", i, keyEntry.ID, time.Now(), keyEntry)
if err != nil {
log.Println("Failed to write key", keyEntry.ID, ":", err)
continue
}
}
}
}
func writeJsonFile(zw *zip.Writer, model string, idx int, id string, t time.Time, data interface{}) error {
w, err := zw.CreateHeader(&zip.FileHeader{
Name: fmt.Sprintf("rpdata_dump_v1/%s/%06d_%s.json", model, idx, id),
Modified: t,
})
if err != nil {
return err
}
return json.NewEncoder(w).Encode(data)
}
database/mongodb/db.go
+4
View File
@@ -11,6 +11,8 @@ import (
"github.com/globalsign/mgo"
)
var DisableFixes bool
type MongoDB struct {
session *mgo.Session
@@ -169,7 +171,9 @@ func Init(cfg config.Database) (*MongoDB, error) {
return nil, err
}
if !DisableFixes {
go posts.fixPositions(logs)
}
return &MongoDB{
session: session,
database/mongodb/posts.go
+6 -1
View File
@@ -91,7 +91,12 @@ func (r *postRepository) List(ctx context.Context, filter models.PostFilter) ([]
}
posts := make([]*models.Post, 0, 32)
err := r.posts.Find(query).Sort("-logId", "position").Limit(filter.Limit).All(&posts)
var err error
if filter.LogID != nil {
err = r.posts.Find(query).Sort("position").Limit(filter.Limit).All(&posts)
} else {
err = r.posts.Find(query).Limit(filter.Limit).All(&posts)
}
if err != nil {
if err == mgo.ErrNotFound {
return []*models.Post{}, nil
database/mongodb/users.go
+11
View File
@@ -5,6 +5,7 @@ import (
"git.aiterp.net/rpdata/api/models"
"git.aiterp.net/rpdata/api/repositories"
"github.com/globalsign/mgo"
"github.com/globalsign/mgo/bson"
)
type userRepository struct {
@@ -31,6 +32,16 @@ func (r *userRepository) Find(ctx context.Context, id string) (*models.User, err
return user, nil
}
func (r *userRepository) List(ctx context.Context) ([]*models.User, error) {
users := make([]*models.User, 0, 8)
err := r.users.Find(bson.M{}).All(&users)
if err != nil && err != mgo.ErrNotFound {
return nil, err
}
return users, nil
}
func (r *userRepository) Insert(ctx context.Context, user models.User) (*models.User, error) {
err := r.users.Insert(user)
if err != nil {
go.mod
+3
View File
@@ -9,6 +9,7 @@ require (
github.com/globalsign/mgo v0.0.0-20180403085842-f76e4f9da92e
github.com/go-ini/ini v1.35.0 // indirect
github.com/golang/protobuf v1.3.1 // indirect
github.com/golang/snappy v0.0.2 // indirect
github.com/google/go-cmp v0.3.0 // indirect
github.com/gorilla/websocket v1.4.0 // indirect
github.com/h2non/filetype v1.0.8
@@ -23,6 +24,8 @@ require (
github.com/tidwall/pretty v1.0.0 // indirect
github.com/vektah/dataloaden v0.3.0
github.com/vektah/gqlparser v1.1.2
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c // indirect
github.com/xdg/stringprep v1.0.0 // indirect
go.mongodb.org/mongo-driver v1.0.3
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 // indirect
golang.org/x/net v0.0.0-20190514140710-3ec191127204 // indirect
go.sum
+6
View File
@@ -29,6 +29,8 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/snappy v0.0.2 h1:aeE13tS0IiQgFjYdoL8qN3K1N2bXXtI6Vi51/y7BpMw=
github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
@@ -104,6 +106,10 @@ github.com/vektah/dataloaden v0.3.0 h1:ZfVN2QD6swgvp+tDqdH/OIT/wu3Dhu0cus0k5gIZS
github.com/vektah/dataloaden v0.3.0/go.mod h1:/HUdMve7rvxZma+2ZELQeNh88+003LL7Pf/CZ089j8U=
github.com/vektah/gqlparser v1.1.2 h1:ZsyLGn7/7jDNI+y4SEhI4yAxRChlv15pUHMjijT+e68=
github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk=
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0=
github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
go.mongodb.org/mongo-driver v1.0.3 h1:GKoji1ld3tw2aC+GX1wbr/J2fX13yNacEYoJ8Nhr0yU=
go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I=
repositories/user.go
+1
View File
@@ -7,5 +7,6 @@ import (
type UserRepository interface {
Find(ctx context.Context, id string) (*models.User, error)
List(ctx context.Context) ([]*models.User, error)
Insert(ctx context.Context, user models.User) (*models.User, error)
}