add rpdata-restore command.

2021-03-21 12:19:27 +01:00
parent c9084287a4
commit e9b00b61fd
13 changed files with 497 additions and 86 deletions
@@ -24,6 +24,8 @@ var flagPassword = flag.String("password", "", "")
 var flagMechanism = flag.String("mechanism", "", "")
 var flagOutputFile = flag.String("outfile", "dump.zip", "The file to write to.")
 var flagIncludeKeys = flag.Bool("include-keys", false, "Whether to include the keys.")
+var flagIncludeUsers = flag.Bool("include-users", false, "Whether to include the keys.")
+var flagIncludeUnlisted = flag.Bool("include-unlisted", false, "Whether to include unlisted stuff.")

 func main() {
 	flag.Parse()
@@ -41,6 +43,8 @@ func main() {
 		Mechanism: *flagMechanism,
 	}

+	blacklist := make(map[string]bool)
+
 	mongodb.DisableFixes = true

 	db, err := database.Init(cfg)
@@ -105,6 +109,10 @@ func main() {
 		log.Println("No changes to dump.")
 	}
 	for i, change := range changes {
+		if !*flagIncludeUnlisted && !change.Listed {
+			continue
+		}
+
 		err := writeJsonFile(zipWriter, "change", i, change.ID, change.Date, change)
 		if err != nil {
 			log.Println("Failed to write change", change.ID, ":", err)
@@ -127,6 +135,11 @@ func main() {
 		log.Println("No stories to dump.")
 	}
 	for i, story := range stories {
+		if !*flagIncludeUnlisted && !story.Listed {
+			blacklist[story.ID] = true
+			continue
+		}
+
 		err := writeJsonFile(zipWriter, "story", i, story.ID, story.CreatedDate, story)
 		if err != nil {
 			log.Println("Failed to write story", story.ID, ":", err)
@@ -143,6 +156,11 @@ func main() {
 		log.Println("No chapters to dump.")
 	}
 	for i, chapter := range chapters {
+		if !*flagIncludeUnlisted && blacklist[chapter.StoryID] {
+			blacklist[chapter.ID] = true
+			continue
+		}
+
 		err := writeJsonFile(zipWriter, "chapter", i, chapter.ID, chapter.CreatedDate, chapter)
 		if err != nil {
 			log.Println("Failed to write chapter", chapter.ID, ":", err)
@@ -159,6 +177,10 @@ func main() {
 		log.Println("No comments to dump.")
 	}
 	for i, comment := range comments {
+		if !*flagIncludeUnlisted && blacklist[comment.ChapterID] {
+			continue
+		}
+
 		err := writeJsonFile(zipWriter, "comment", i, comment.ID, comment.CreatedDate, comment)
 		if err != nil {
 			log.Println("Failed to write comment", comment.ID, ":", err)
@@ -191,26 +213,28 @@ func main() {
 		log.Println("No logs to dump.")
 	}
 	for i, logEntry := range logs {
-		err := writeJsonFile(zipWriter, "post", i, logEntry.ID, logEntry.Date, logEntry)
+		err := writeJsonFile(zipWriter, "log", i, logEntry.ID, logEntry.Date, logEntry)
 		if err != nil {
 			log.Println("Failed to write post", logEntry.ID, ":", err)
 			continue
 		}
 	}

-	log.Println("Dumping users...")
-	users, err := db.Users().List(ctx)
-	if err != nil {
-		log.Println("Failed to get users:", err)
-	}
-	if len(users) == 0 {
-		log.Println("No users to dump.")
-	}
-	for i, userEntry := range users {
-		err := writeJsonFile(zipWriter, "user", i, userEntry.ID, time.Now(), userEntry)
+	if *flagIncludeUsers {
+		log.Println("Dumping users...")
+		users, err := db.Users().List(ctx)
 		if err != nil {
-			log.Println("Failed to write user", userEntry.ID, ":", err)
-			continue
+			log.Println("Failed to get users:", err)
+		}
+		if len(users) == 0 {
+			log.Println("No users to dump.")
+		}
+		for i, userEntry := range users {
+			err := writeJsonFile(zipWriter, "user", i, userEntry.ID, time.Now(), userEntry)
+			if err != nil {
+				log.Println("Failed to write user", userEntry.ID, ":", err)
+				continue
+			}
 		}
 	}

@@ -0,0 +1,270 @@
+package main
+
+import (
+	"archive/zip"
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"git.aiterp.net/rpdata/api/database"
+	"git.aiterp.net/rpdata/api/database/mongodb"
+	"git.aiterp.net/rpdata/api/internal/config"
+	"git.aiterp.net/rpdata/api/models"
+	"log"
+	"strings"
+	"time"
+)
+
+var flagDriver = flag.String("driver", "mongodb", "The database driver to use.")
+var flagHost = flag.String("host", "127.0.0.1", "The host to connect to.")
+var flagPort = flag.Int("port", 27017, "The port to connect on.")
+var flagDb = flag.String("db", "rpdata", "The database name")
+var flagUsername = flag.String("username", "", "")
+var flagPassword = flag.String("password", "", "")
+var flagMechanism = flag.String("mechanism", "", "")
+var flagInputFile = flag.String("infile", "dump.zip", "The file to read from.")
+var flagIncludeKeys = flag.Bool("include-keys", false, "Whether to include the keys.")
+
+func main() {
+	flag.Parse()
+
+	cfg := config.Database{
+		Driver:     *flagDriver,
+		Host:       *flagHost,
+		Port:       *flagPort,
+		Db:         *flagDb,
+		Username:   *flagUsername,
+		Password:   *flagPassword,
+		Mechanism:  *flagMechanism,
+		RestoreIDs: true,
+	}
+
+	mongodb.DisableFixes = true
+
+	db, err := database.Init(cfg)
+	if err != nil {
+		log.Fatalln("Failed to open database:", err)
+	}
+
+	cfg2 := cfg
+	cfg2.RestoreIDs = false
+
+	db.Tags()
+
+	zipReader, err := zip.OpenReader(*flagInputFile)
+	if err != nil {
+		log.Fatalln("Failed to open input file:", err)
+	}
+	defer func() {
+		err = zipReader.Close()
+		if err != nil {
+			log.Fatalln("Failed to close input file:", err)
+		}
+	}()
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Minute*30)
+	defer cancel()
+
+	postMap := make(map[string][]*models.Post)
+
+	for _, file := range zipReader.File {
+		if strings.HasSuffix(file.Name, "/") {
+			continue
+		}
+
+		parts := strings.Split(file.Name, "/")
+		if len(parts) < 3 || parts[0] != "rpdata_dump_v1" {
+			log.Fatalln("Unrecognized file path:", file.Name)
+		}
+
+		reader, err := file.Open()
+		if err != nil {
+			log.Fatalln("Unrecognized file:", file.Name, err)
+		}
+
+		hideList := make(map[string]bool)
+
+		switch parts[1] {
+		case "character":
+			{
+				character := models.Character{}
+				err := json.NewDecoder(reader).Decode(&character)
+				if err != nil {
+					log.Fatalln("Could not parse character:", parts[2], err)
+				}
+
+				_, err = db.Characters().Insert(ctx, character)
+				if err != nil {
+					log.Fatalln("Could not insert character:", parts[2], err)
+				}
+
+				log.Println("Character", character.Name, "inserted.")
+			}
+		case "channel":
+			{
+				channel := models.Channel{}
+				err := json.NewDecoder(reader).Decode(&channel)
+				if err != nil {
+					log.Fatalln("Could not parse channel:", parts[2], err)
+				}
+
+				_, err = db.Channels().Insert(ctx, channel)
+				if err != nil {
+					log.Fatalln("Could not insert channel:", parts[2], err)
+				}
+
+				log.Println("Channel", channel.Name, "inserted.")
+			}
+		case "change":
+			{
+				change := models.Change{}
+				err := json.NewDecoder(reader).Decode(&change)
+				if err != nil {
+					log.Fatalln("Could not parse character:", parts[2], err)
+				}
+
+				_, err = db.Changes().Insert(ctx, change)
+				if err != nil {
+					log.Fatalln("Could not insert character:", parts[2], err)
+				}
+
+				if change.Listed {
+					log.Println("Change", change.ID, "inserted.")
+				} else {
+					log.Println("Unlisted change inserted.")
+				}
+			}
+		case "story":
+			{
+				story := models.Story{}
+				err := json.NewDecoder(reader).Decode(&story)
+				if err != nil {
+					log.Fatalln("Could not parse story:", parts[2], err)
+				}
+
+				_, err = db.Stories().Insert(ctx, story)
+				if err != nil {
+					log.Fatalln("Could not insert story:", parts[2], err)
+				}
+
+				if story.Listed {
+					log.Println("Story", story.Name, "inserted.")
+				} else {
+					log.Println("Unlisted story inserted.")
+					hideList[story.ID] = true
+				}
+			}
+		case "chapter":
+			{
+				chapter := models.Chapter{}
+				err := json.NewDecoder(reader).Decode(&chapter)
+				if err != nil {
+					log.Fatalln("Could not parse story:", parts[2], err)
+				}
+
+				_, err = db.Chapters().Insert(ctx, chapter)
+				if err != nil {
+					log.Fatalln("Could not insert story:", parts[2], err)
+				}
+
+				if !hideList[chapter.StoryID] {
+					log.Println("Chapter", fmt.Sprintf("%s (id: %s)", chapter.Title, chapter.ID), "inserted.")
+				} else {
+					log.Println("Unlisted chapter inserted.")
+					hideList[chapter.ID] = true
+				}
+			}
+		case "comment":
+			{
+				comment := models.Comment{}
+				err := json.NewDecoder(reader).Decode(&comment)
+				if err != nil {
+					log.Fatalln("Could not parse story:", parts[2], err)
+				}
+
+				_, err = db.Comments().Insert(ctx, comment)
+				if err != nil {
+					log.Fatalln("Could not insert story:", parts[2], err)
+				}
+
+				if !hideList[comment.ChapterID] {
+					log.Println("Comment", comment.Subject, "inserted.")
+				} else {
+					log.Println("Unlisted comment inserted.")
+				}
+			}
+
+		case "log":
+			{
+				logg := models.Log{}
+				err := json.NewDecoder(reader).Decode(&logg)
+				if err != nil {
+					log.Fatalln("Could not parse log:", parts[2], err)
+				}
+
+				_, err = db.Logs().Insert(ctx, logg)
+				if err != nil {
+					log.Fatalln("Could not insert log:", parts[2], err)
+				}
+
+				log.Println("Log", logg.Date.Format(time.RFC3339)[:16], logg.ChannelName, "inserted.")
+			}
+		case "post":
+			{
+				post := models.Post{}
+				err := json.NewDecoder(reader).Decode(&post)
+				if err != nil {
+					log.Fatalln("Could not parse post:", parts[2], err)
+				}
+
+				postMap[post.LogID] = append(postMap[post.LogID], &post)
+			}
+
+		case "user":
+			{
+				user := models.User{}
+				err := json.NewDecoder(reader).Decode(&user)
+				if err != nil {
+					log.Fatalln("Could not parse post:", parts[2], err)
+				}
+
+				_, err = db.Users().Insert(ctx, user)
+				if err != nil {
+					log.Fatalln("Could not insert user:", parts[2], err)
+				}
+
+				log.Println("User", user.ID, "inserted.")
+			}
+		case "key":
+			{
+				if !*flagIncludeKeys {
+					break
+				}
+
+				key := models.Key{}
+				err := json.NewDecoder(reader).Decode(&key)
+				if err != nil {
+					log.Fatalln("Could not parse key:", parts[2], err)
+				}
+
+				_, err = db.Keys().Insert(ctx, key)
+				if err != nil {
+					log.Fatalln("Could not insert key:", parts[2], err)
+				}
+
+				log.Println("Key", key.ID, "inserted.")
+			}
+		}
+
+		reader.Close()
+	}
+
+	for _, posts := range postMap {
+		_, err = db.Posts().InsertMany(ctx, posts...)
+		if err != nil {
+			log.Fatalln("Could not insert post for logId:", posts[0].LogID, err)
+		}
+
+		log.Printf("Inserted %d posts for log %s.", len(posts), posts[0].LogID)
+	}
+}