add dump/restore commands to docker file.

.drone.yml

@@ -1,21 +1,39 @@
-workspace:
-  base: /go
-  path: src/git.aiterp.net/rpdata/api
+name: rpdata-api
 
-pipeline:
-  docker-release:
+kind: pipeline
+type: docker
+
+steps:
+  - name: go-test
+    image: golang:1.14
+    commands:
+      - go generate ./...
+      - go test -v ./...
+
+  - name: docker-release-tag
     image: plugins/docker
-    repo: vmaple/rpdata
-    auto_tag: true
-    secrets: [ docker_username, docker_password ]
+    settings:
+      auto_tag: true
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
+      repo: r.vmaple.dev/rpdata/rpdata-api
+      registry: r.vmaple.dev
     when:
-      event: tag
-      ref: refs/tags/*
-  docker-release-latest:
+      event:
+        - tag
+
+  - name: docker-release-latest
     image: plugins/docker
-    repo: vmaple/rpdata
-    tags: latest
-    secrets: [ docker_username, docker_password ]
+    settings:
+      auto_tag: false
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
+      repo: r.vmaple.dev/rpdata/rpdata-api
+      registry: r.vmaple.dev
     when:
-      event: tag
-      ref: refs/tags/*
+      event:
+        - tag

.gitignore

@@ -10,6 +10,11 @@ rpdata-*
 .vscode
 .idea
 
+/main
+/main.exe
+
 /graph2/graphcore/*_gen.go
 generated.gql
 generated.go
+/dump.zip
+/rpdata-dump-v1/

Dockerfile

@@ -1,6 +1,6 @@
 ## 1. Build Server
-# Use Go 1.12
-FROM golang:1.12 AS build-server
+# Use Go 1.14
+FROM golang:1.16 AS build-server
 
 # Load repository into docker
 WORKDIR /project/
@@ -12,10 +12,12 @@ RUN go generate ./...
 # Build server and tools (CGO disabled to make them 100% static)
 ENV CGO_ENABLED 0
 RUN go build -installsuffix cgo -ldflags="-s -w" -o /binaries/rpdata-server ./cmd/rpdata-server
+RUN go build -installsuffix cgo -ldflags="-s -w" -o /binaries/rpdata-dump ./cmd/rpdata-dump
+RUN go build -installsuffix cgo -ldflags="-s -w" -o /binaries/rpdata-restore ./cmd/rpdata-restore
 
 ## 2. Distribute
 # Use alpine linux
-FROM alpine:3.7
+FROM alpine:3.13
 
 # Install certificates to enable HTTPS GET requests, and tzdata for log imports.
 RUN apk add --no-cache ca-certificates tzdata

cmd/rpdata-dump/main.go

@@ -0,0 +1,270 @@
+package main
+
+import (
+	"archive/zip"
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"git.aiterp.net/rpdata/api/database"
+	"git.aiterp.net/rpdata/api/database/mongodb"
+	"git.aiterp.net/rpdata/api/internal/config"
+	"git.aiterp.net/rpdata/api/models"
+	"log"
+	"os"
+	"time"
+)
+
+var flagDriver = flag.String("driver", "mongodb", "The database driver to use.")
+var flagHost = flag.String("host", "127.0.0.1", "The host to connect to.")
+var flagPort = flag.Int("port", 27017, "The port to connect on.")
+var flagDb = flag.String("db", "rpdata", "The database name")
+var flagUsername = flag.String("username", "", "")
+var flagPassword = flag.String("password", "", "")
+var flagMechanism = flag.String("mechanism", "", "")
+var flagOutputFile = flag.String("outfile", "dump.zip", "The file to write to.")
+var flagIncludeKeys = flag.Bool("include-keys", false, "Whether to include the keys.")
+var flagIncludeUsers = flag.Bool("include-users", false, "Whether to include the keys.")
+var flagIncludeUnlisted = flag.Bool("include-unlisted", false, "Whether to include unlisted stuff.")
+
+func main() {
+	flag.Parse()
+
+	trueValue := true
+	truePtr := &trueValue
+
+	cfg := config.Database{
+		Driver:    *flagDriver,
+		Host:      *flagHost,
+		Port:      *flagPort,
+		Db:        *flagDb,
+		Username:  *flagUsername,
+		Password:  *flagPassword,
+		Mechanism: *flagMechanism,
+	}
+
+	blacklist := make(map[string]bool)
+
+	mongodb.DisableFixes = true
+
+	db, err := database.Init(cfg)
+	if err != nil {
+		log.Fatalln("Failed to open database:", err)
+	}
+
+	file, err := os.OpenFile(*flagOutputFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
+	if err != nil {
+		log.Fatalln("Failed to open output file:", err)
+	}
+
+	zipWriter := zip.NewWriter(file)
+	defer func() {
+		err = zipWriter.Close()
+		if err != nil {
+			log.Fatalln("Failed to close output file:", err)
+		}
+	}()
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Minute*30)
+	defer cancel()
+
+	log.Println("Dumping characters...")
+	characters, err := db.Characters().List(ctx, models.CharacterFilter{})
+	if err != nil {
+		log.Println("Failed to get characters:", err)
+	}
+	if len(characters) == 0 {
+		log.Println("No characters to dump.")
+	}
+	for i, character := range characters {
+		err := writeJsonFile(zipWriter, "character", i, character.ID, time.Now(), character)
+		if err != nil {
+			log.Println("Failed to write character", character.ID, ":", err)
+			continue
+		}
+	}
+
+	log.Println("Dumping channels...")
+	channels, err := db.Channels().List(ctx, models.ChannelFilter{})
+	if err != nil {
+		log.Println("Failed to get channels:", err)
+	}
+	if len(channels) == 0 {
+		log.Println("No channels to dump.")
+	}
+	for i, channel := range channels {
+		err := writeJsonFile(zipWriter, "channel", i, channel.Name, time.Now(), channel)
+		if err != nil {
+			log.Println("Failed to write channel", channel.Name, ":", err)
+			continue
+		}
+	}
+
+	log.Println("Dumping changes...")
+	changes, err := db.Changes().List(ctx, models.ChangeFilter{})
+	if err != nil {
+		log.Println("Failed to get changes:", err)
+	}
+	if len(changes) == 0 {
+		log.Println("No changes to dump.")
+	}
+	for i, change := range changes {
+		if !*flagIncludeUnlisted && !change.Listed {
+			continue
+		}
+
+		err := writeJsonFile(zipWriter, "change", i, change.ID, change.Date, change)
+		if err != nil {
+			log.Println("Failed to write change", change.ID, ":", err)
+			continue
+		}
+	}
+
+	log.Println("Dumping stories...")
+	stories, err := db.Stories().List(ctx, models.StoryFilter{})
+	if err != nil {
+		log.Println("Failed to get stories:", err)
+	}
+	unlistedStories, err := db.Stories().List(ctx, models.StoryFilter{Unlisted: truePtr})
+	if err != nil {
+		log.Println("Failed to get unlisted stories:", err)
+	} else {
+		stories = append(stories, unlistedStories...)
+	}
+	if len(stories) == 0 {
+		log.Println("No stories to dump.")
+	}
+	for i, story := range stories {
+		if !*flagIncludeUnlisted && !story.Listed {
+			blacklist[story.ID] = true
+			continue
+		}
+
+		err := writeJsonFile(zipWriter, "story", i, story.ID, story.CreatedDate, story)
+		if err != nil {
+			log.Println("Failed to write story", story.ID, ":", err)
+			continue
+		}
+	}
+
+	log.Println("Dumping chapters...")
+	chapters, err := db.Chapters().List(ctx, models.ChapterFilter{})
+	if err != nil {
+		log.Println("Failed to get chapters:", err)
+	}
+	if len(chapters) == 0 {
+		log.Println("No chapters to dump.")
+	}
+	for i, chapter := range chapters {
+		if !*flagIncludeUnlisted && blacklist[chapter.StoryID] {
+			blacklist[chapter.ID] = true
+			continue
+		}
+
+		err := writeJsonFile(zipWriter, "chapter", i, chapter.ID, chapter.CreatedDate, chapter)
+		if err != nil {
+			log.Println("Failed to write chapter", chapter.ID, ":", err)
+			continue
+		}
+	}
+
+	log.Println("Dumping comments...")
+	comments, err := db.Comments().List(ctx, models.CommentFilter{})
+	if err != nil {
+		log.Println("Failed to get comments:", err)
+	}
+	if len(comments) == 0 {
+		log.Println("No comments to dump.")
+	}
+	for i, comment := range comments {
+		if !*flagIncludeUnlisted && blacklist[comment.ChapterID] {
+			continue
+		}
+
+		err := writeJsonFile(zipWriter, "comment", i, comment.ID, comment.CreatedDate, comment)
+		if err != nil {
+			log.Println("Failed to write comment", comment.ID, ":", err)
+			continue
+		}
+	}
+
+	log.Println("Dumping posts...")
+	posts, err := db.Posts().List(ctx, models.PostFilter{})
+	if err != nil {
+		log.Println("Failed to get posts:", err)
+	}
+	if len(posts) == 0 {
+		log.Println("No posts to dump.")
+	}
+	for i, post := range posts {
+		err := writeJsonFile(zipWriter, "post", i, post.ID, post.Time, post)
+		if err != nil {
+			log.Println("Failed to write post", post.ID, ":", err)
+			continue
+		}
+	}
+
+	log.Println("Dumping logs...")
+	logs, err := db.Logs().List(ctx, models.LogFilter{})
+	if err != nil {
+		log.Println("Failed to get logs:", err)
+	}
+	if len(logs) == 0 {
+		log.Println("No logs to dump.")
+	}
+	for i, logEntry := range logs {
+		err := writeJsonFile(zipWriter, "log", i, logEntry.ID, logEntry.Date, logEntry)
+		if err != nil {
+			log.Println("Failed to write post", logEntry.ID, ":", err)
+			continue
+		}
+	}
+
+	if *flagIncludeUsers {
+		log.Println("Dumping users...")
+		users, err := db.Users().List(ctx)
+		if err != nil {
+			log.Println("Failed to get users:", err)
+		}
+		if len(users) == 0 {
+			log.Println("No users to dump.")
+		}
+		for i, userEntry := range users {
+			err := writeJsonFile(zipWriter, "user", i, userEntry.ID, time.Now(), userEntry)
+			if err != nil {
+				log.Println("Failed to write user", userEntry.ID, ":", err)
+				continue
+			}
+		}
+	}
+
+	if *flagIncludeKeys {
+		log.Println("Dumping keys...")
+		keys, err := db.Keys().List(ctx, models.KeyFilter{})
+		if err != nil {
+			log.Println("Failed to get users:", err)
+		}
+		if len(keys) == 0 {
+			log.Println("No users to dump.")
+		}
+		for i, keyEntry := range keys {
+			err := writeJsonFile(zipWriter, "key", i, keyEntry.ID, time.Now(), keyEntry)
+			if err != nil {
+				log.Println("Failed to write key", keyEntry.ID, ":", err)
+				continue
+			}
+		}
+	}
+}
+
+func writeJsonFile(zw *zip.Writer, model string, idx int, id string, t time.Time, data interface{}) error {
+	w, err := zw.CreateHeader(&zip.FileHeader{
+		Name:     fmt.Sprintf("rpdata_dump_v1/%s/%06d_%s.json", model, idx, id),
+		Modified: t,
+	})
+	if err != nil {
+		return err
+	}
+
+	return json.NewEncoder(w).Encode(data)
+}

cmd/rpdata-restore/main.go

@@ -0,0 +1,270 @@
+package main
+
+import (
+	"archive/zip"
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"git.aiterp.net/rpdata/api/database"
+	"git.aiterp.net/rpdata/api/database/mongodb"
+	"git.aiterp.net/rpdata/api/internal/config"
+	"git.aiterp.net/rpdata/api/models"
+	"log"
+	"strings"
+	"time"
+)
+
+var flagDriver = flag.String("driver", "mongodb", "The database driver to use.")
+var flagHost = flag.String("host", "127.0.0.1", "The host to connect to.")
+var flagPort = flag.Int("port", 27017, "The port to connect on.")
+var flagDb = flag.String("db", "rpdata", "The database name")
+var flagUsername = flag.String("username", "", "")
+var flagPassword = flag.String("password", "", "")
+var flagMechanism = flag.String("mechanism", "", "")
+var flagInputFile = flag.String("infile", "dump.zip", "The file to read from.")
+var flagIncludeKeys = flag.Bool("include-keys", false, "Whether to include the keys.")
+
+func main() {
+	flag.Parse()
+
+	cfg := config.Database{
+		Driver:     *flagDriver,
+		Host:       *flagHost,
+		Port:       *flagPort,
+		Db:         *flagDb,
+		Username:   *flagUsername,
+		Password:   *flagPassword,
+		Mechanism:  *flagMechanism,
+		RestoreIDs: true,
+	}
+
+	mongodb.DisableFixes = true
+
+	db, err := database.Init(cfg)
+	if err != nil {
+		log.Fatalln("Failed to open database:", err)
+	}
+
+	cfg2 := cfg
+	cfg2.RestoreIDs = false
+
+	db.Tags()
+
+	zipReader, err := zip.OpenReader(*flagInputFile)
+	if err != nil {
+		log.Fatalln("Failed to open input file:", err)
+	}
+	defer func() {
+		err = zipReader.Close()
+		if err != nil {
+			log.Fatalln("Failed to close input file:", err)
+		}
+	}()
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Minute*30)
+	defer cancel()
+
+	postMap := make(map[string][]*models.Post)
+
+	for _, file := range zipReader.File {
+		if strings.HasSuffix(file.Name, "/") {
+			continue
+		}
+
+		parts := strings.Split(file.Name, "/")
+		if len(parts) < 3 || parts[0] != "rpdata_dump_v1" {
+			log.Fatalln("Unrecognized file path:", file.Name)
+		}
+
+		reader, err := file.Open()
+		if err != nil {
+			log.Fatalln("Unrecognized file:", file.Name, err)
+		}
+
+		hideList := make(map[string]bool)
+
+		switch parts[1] {
+		case "character":
+			{
+				character := models.Character{}
+				err := json.NewDecoder(reader).Decode(&character)
+				if err != nil {
+					log.Fatalln("Could not parse character:", parts[2], err)
+				}
+
+				_, err = db.Characters().Insert(ctx, character)
+				if err != nil {
+					log.Fatalln("Could not insert character:", parts[2], err)
+				}
+
+				log.Println("Character", character.Name, "inserted.")
+			}
+		case "channel":
+			{
+				channel := models.Channel{}
+				err := json.NewDecoder(reader).Decode(&channel)
+				if err != nil {
+					log.Fatalln("Could not parse channel:", parts[2], err)
+				}
+
+				_, err = db.Channels().Insert(ctx, channel)
+				if err != nil {
+					log.Fatalln("Could not insert channel:", parts[2], err)
+				}
+
+				log.Println("Channel", channel.Name, "inserted.")
+			}
+		case "change":
+			{
+				change := models.Change{}
+				err := json.NewDecoder(reader).Decode(&change)
+				if err != nil {
+					log.Fatalln("Could not parse character:", parts[2], err)
+				}
+
+				_, err = db.Changes().Insert(ctx, change)
+				if err != nil {
+					log.Fatalln("Could not insert character:", parts[2], err)
+				}
+
+				if change.Listed {
+					log.Println("Change", change.ID, "inserted.")
+				} else {
+					log.Println("Unlisted change inserted.")
+				}
+			}
+		case "story":
+			{
+				story := models.Story{}
+				err := json.NewDecoder(reader).Decode(&story)
+				if err != nil {
+					log.Fatalln("Could not parse story:", parts[2], err)
+				}
+
+				_, err = db.Stories().Insert(ctx, story)
+				if err != nil {
+					log.Fatalln("Could not insert story:", parts[2], err)
+				}
+
+				if story.Listed {
+					log.Println("Story", story.Name, "inserted.")
+				} else {
+					log.Println("Unlisted story inserted.")
+					hideList[story.ID] = true
+				}
+			}
+		case "chapter":
+			{
+				chapter := models.Chapter{}
+				err := json.NewDecoder(reader).Decode(&chapter)
+				if err != nil {
+					log.Fatalln("Could not parse story:", parts[2], err)
+				}
+
+				_, err = db.Chapters().Insert(ctx, chapter)
+				if err != nil {
+					log.Fatalln("Could not insert story:", parts[2], err)
+				}
+
+				if !hideList[chapter.StoryID] {
+					log.Println("Chapter", fmt.Sprintf("%s (id: %s)", chapter.Title, chapter.ID), "inserted.")
+				} else {
+					log.Println("Unlisted chapter inserted.")
+					hideList[chapter.ID] = true
+				}
+			}
+		case "comment":
+			{
+				comment := models.Comment{}
+				err := json.NewDecoder(reader).Decode(&comment)
+				if err != nil {
+					log.Fatalln("Could not parse story:", parts[2], err)
+				}
+
+				_, err = db.Comments().Insert(ctx, comment)
+				if err != nil {
+					log.Fatalln("Could not insert story:", parts[2], err)
+				}
+
+				if !hideList[comment.ChapterID] {
+					log.Println("Comment", comment.Subject, "inserted.")
+				} else {
+					log.Println("Unlisted comment inserted.")
+				}
+			}
+
+		case "log":
+			{
+				logg := models.Log{}
+				err := json.NewDecoder(reader).Decode(&logg)
+				if err != nil {
+					log.Fatalln("Could not parse log:", parts[2], err)
+				}
+
+				_, err = db.Logs().Insert(ctx, logg)
+				if err != nil {
+					log.Fatalln("Could not insert log:", parts[2], err)
+				}
+
+				log.Println("Log", logg.Date.Format(time.RFC3339)[:16], logg.ChannelName, "inserted.")
+			}
+		case "post":
+			{
+				post := models.Post{}
+				err := json.NewDecoder(reader).Decode(&post)
+				if err != nil {
+					log.Fatalln("Could not parse post:", parts[2], err)
+				}
+
+				postMap[post.LogID] = append(postMap[post.LogID], &post)
+			}
+
+		case "user":
+			{
+				user := models.User{}
+				err := json.NewDecoder(reader).Decode(&user)
+				if err != nil {
+					log.Fatalln("Could not parse post:", parts[2], err)
+				}
+
+				_, err = db.Users().Insert(ctx, user)
+				if err != nil {
+					log.Fatalln("Could not insert user:", parts[2], err)
+				}
+
+				log.Println("User", user.ID, "inserted.")
+			}
+		case "key":
+			{
+				if !*flagIncludeKeys {
+					break
+				}
+
+				key := models.Key{}
+				err := json.NewDecoder(reader).Decode(&key)
+				if err != nil {
+					log.Fatalln("Could not parse key:", parts[2], err)
+				}
+
+				_, err = db.Keys().Insert(ctx, key)
+				if err != nil {
+					log.Fatalln("Could not insert key:", parts[2], err)
+				}
+
+				log.Println("Key", key.ID, "inserted.")
+			}
+		}
+
+		reader.Close()
+	}
+
+	for _, posts := range postMap {
+		_, err = db.Posts().InsertMany(ctx, posts...)
+		if err != nil {
+			log.Fatalln("Could not insert post for logId:", posts[0].LogID, err)
+		}
+
+		log.Printf("Inserted %d posts for log %s.", len(posts), posts[0].LogID)
+	}
+}

database/mongodb/changes.go

@@ -2,17 +2,20 @@ package mongodb
 
 import (
 	"context"
+	"errors"
 	"git.aiterp.net/rpdata/api/models"
 	"git.aiterp.net/rpdata/api/repositories"
 	"github.com/globalsign/mgo"
 	"github.com/globalsign/mgo/bson"
 	"strconv"
+	"strings"
 	"time"
 )
 
 type changeRepository struct {
-	changes   *mgo.Collection
-	idCounter *counter
+	restoreIDs bool
+	changes    *mgo.Collection
+	idCounter  *counter
 }
 
 func (r *changeRepository) Find(ctx context.Context, id string) (*models.Change, error) {
@@ -33,7 +36,7 @@ func (r *changeRepository) List(ctx context.Context, filter models.ChangeFilter)
 	}
 	if len(filter.Keys) > 0 {
 		query["keys"] = bson.M{"$in": filter.Keys}
-	} else {
+	} else if !r.restoreIDs {
 		query["listed"] = true
 	}
 	if filter.Author != nil && *filter.Author != "" {
@@ -58,14 +61,28 @@ func (r *changeRepository) List(ctx context.Context, filter models.ChangeFilter)
 }
 
 func (r *changeRepository) Insert(ctx context.Context, change models.Change) (*models.Change, error) {
-	next, err := r.idCounter.Increment(1)
-	if err != nil {
-		return nil, err
+	if !r.restoreIDs || change.ID == "" {
+		next, err := r.idCounter.Increment(1)
+		if err != nil {
+			return nil, err
+		}
+
+		change.ID = "Change_" + strconv.Itoa(next)
+	} else {
+		tokens := strings.Split(change.ID, "_")
+		if len(tokens) != 2 || tokens[0] != "Change" {
+			return nil, errors.New("Invalid change ID")
+		}
+
+		n, err := strconv.Atoi(tokens[1])
+		if err != nil {
+			return nil, err
+		}
+
+		_ = r.idCounter.Bump(n)
 	}
 
-	change.ID = "Change_" + strconv.Itoa(next)
-
-	err = r.changes.Insert(&change)
+	err := r.changes.Insert(&change)
 	if err != nil {
 		return nil, err
 	}
@@ -77,7 +94,7 @@ func (r *changeRepository) Remove(ctx context.Context, change models.Change) err
 	return r.changes.RemoveId(change.ID)
 }
 
-func newChangeRepository(db *mgo.Database) (repositories.ChangeRepository, error) {
+func newChangeRepository(db *mgo.Database, restoreIDs bool) (repositories.ChangeRepository, error) {
 	collection := db.C("common.changes")
 
 	// Delete the old index if it exists.
@@ -103,7 +120,8 @@ func newChangeRepository(db *mgo.Database) (repositories.ChangeRepository, error
 	}
 
 	return &changeRepository{
-		changes:   collection,
-		idCounter: newCounter(db, "auto_increment", "Change"),
+		restoreIDs: restoreIDs,
+		changes:    collection,
+		idCounter:  newCounter(db, "auto_increment", "Change"),
 	}, nil
 }

database/mongodb/chapters.go

@@ -2,20 +2,23 @@ package mongodb
 
 import (
 	"context"
+	"errors"
 	"git.aiterp.net/rpdata/api/internal/generate"
 	"git.aiterp.net/rpdata/api/models"
 	"git.aiterp.net/rpdata/api/repositories"
 	"github.com/globalsign/mgo"
 	"github.com/globalsign/mgo/bson"
 	"log"
+	"strings"
 )
 
 type chapterRepository struct {
-	chapters *mgo.Collection
-	comments *mgo.Collection
+	restoreIDs bool
+	chapters   *mgo.Collection
+	comments   *mgo.Collection
 }
 
-func newChapterRepository(db *mgo.Database) (repositories.ChapterRepository, error) {
+func newChapterRepository(db *mgo.Database, restoreIDs bool) (repositories.ChapterRepository, error) {
 	collection := db.C("story.chapters")
 
 	err := collection.EnsureIndexKey("storyId")
@@ -32,8 +35,9 @@ func newChapterRepository(db *mgo.Database) (repositories.ChapterRepository, err
 	}
 
 	return &chapterRepository{
-		chapters: collection,
-		comments: db.C("story.comments"),
+		restoreIDs: restoreIDs,
+		chapters:   collection,
+		comments:   db.C("story.comments"),
 	}, nil
 }
 
@@ -67,7 +71,13 @@ func (r *chapterRepository) List(ctx context.Context, filter models.ChapterFilte
 }
 
 func (r *chapterRepository) Insert(ctx context.Context, chapter models.Chapter) (*models.Chapter, error) {
-	chapter.ID = generate.StoryID()
+	if !r.restoreIDs {
+		chapter.ID = generate.ChapterID()
+	} else {
+		if len(chapter.ID) != len(generate.ChapterID()) && strings.HasPrefix(chapter.ID, "S") {
+			return nil, errors.New("invalid story id")
+		}
+	}
 
 	err := r.chapters.Insert(chapter)
 	if err != nil {

database/mongodb/characters.go

@@ -15,9 +15,10 @@ import (
 type characterRepository struct {
 	characters *mgo.Collection
 	cidCounter *counter
+	restoreIDs bool
 }
 
-func newCharacterRepository(db *mgo.Database) (repositories.CharacterRepository, error) {
+func newCharacterRepository(db *mgo.Database, restoreIDs bool) (repositories.CharacterRepository, error) {
 	collection := db.C("common.characters")
 
 	err := collection.EnsureIndexKey("name")
@@ -52,6 +53,7 @@ func newCharacterRepository(db *mgo.Database) (repositories.CharacterRepository,
 	}
 
 	return &characterRepository{
+		restoreIDs: restoreIDs,
 		characters: collection,
 		cidCounter: newCounter(db, "auto_increment", "Character"),
 	}, nil
@@ -150,13 +152,25 @@ func (r *characterRepository) List(ctx context.Context, filter models.CharacterF
 }
 
 func (r *characterRepository) Insert(ctx context.Context, character models.Character) (*models.Character, error) {
-	nextId, err := r.cidCounter.Increment(1)
-	if err != nil {
-		return nil, err
-	}
-	character.ID = "C" + strconv.Itoa(nextId)
+	if !r.restoreIDs {
+		nextId, err := r.cidCounter.Increment(1)
+		if err != nil {
+			return nil, err
+		}
+		character.ID = "C" + strconv.Itoa(nextId)
+	} else {
+		n, err := strconv.Atoi(character.ID[1:])
+		if err != nil {
+			return nil, err
+		}
 
-	err = r.characters.Insert(&character)
+		err = r.cidCounter.Bump(n)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	err := r.characters.Insert(&character)
 	if err != nil {
 		return nil, err
 	}

database/mongodb/comments.go

@@ -2,19 +2,22 @@ package mongodb
 
 import (
 	"context"
+	"errors"
 	"git.aiterp.net/rpdata/api/internal/generate"
 	"git.aiterp.net/rpdata/api/models"
 	"git.aiterp.net/rpdata/api/repositories"
 	"github.com/globalsign/mgo"
 	"github.com/globalsign/mgo/bson"
 	"log"
+	"strings"
 )
 
 type commentRepository struct {
-	comments *mgo.Collection
+	restoreIDs bool
+	comments   *mgo.Collection
 }
 
-func newCommentRepository(db *mgo.Database) (repositories.CommentRepository, error) {
+func newCommentRepository(db *mgo.Database, restoreIDs bool) (repositories.CommentRepository, error) {
 	collection := db.C("story.comments")
 
 	err := collection.EnsureIndexKey("chapterId")
@@ -31,7 +34,8 @@ func newCommentRepository(db *mgo.Database) (repositories.CommentRepository, err
 	}
 
 	r := &commentRepository{
-		comments: collection,
+		restoreIDs: restoreIDs,
+		comments:   collection,
 	}
 
 	go r.fixFieldTypo()
@@ -69,7 +73,13 @@ func (r *commentRepository) List(ctx context.Context, filter models.CommentFilte
 }
 
 func (r *commentRepository) Insert(ctx context.Context, comment models.Comment) (*models.Comment, error) {
-	comment.ID = generate.CommentID()
+	if !r.restoreIDs {
+		comment.ID = generate.CommentID()
+	} else {
+		if len(comment.ID) != len(generate.CommentID()) && strings.HasPrefix(comment.ID, "SSC") {
+			return nil, errors.New("invalid story id")
+		}
+	}
 
 	err := r.comments.Insert(comment)
 	if err != nil {

database/mongodb/db.go

@@ -11,6 +11,8 @@ import (
 	"github.com/globalsign/mgo"
 )
 
+var DisableFixes bool
+
 type MongoDB struct {
 	session *mgo.Session
 
@@ -103,7 +105,7 @@ func Init(cfg config.Database) (*MongoDB, error) {
 
 	db := session.DB(cfg.Db)
 
-	characters, err := newCharacterRepository(db)
+	characters, err := newCharacterRepository(db, cfg.RestoreIDs)
 	if err != nil {
 		session.Close()
 		return nil, err
@@ -115,19 +117,19 @@ func Init(cfg config.Database) (*MongoDB, error) {
 		return nil, err
 	}
 
-	changes, err := newChangeRepository(db)
+	changes, err := newChangeRepository(db, cfg.RestoreIDs)
 	if err != nil {
 		session.Close()
 		return nil, err
 	}
 
-	logs, err := newLogRepository(db)
+	logs, err := newLogRepository(db, cfg.RestoreIDs)
 	if err != nil {
 		session.Close()
 		return nil, err
 	}
 
-	posts, err := newPostRepository(db)
+	posts, err := newPostRepository(db, cfg.RestoreIDs)
 	if err != nil {
 		session.Close()
 		return nil, err
@@ -139,25 +141,25 @@ func Init(cfg config.Database) (*MongoDB, error) {
 		return nil, err
 	}
 
-	stories, err := newStoryRepository(db)
+	stories, err := newStoryRepository(db, cfg.RestoreIDs)
 	if err != nil {
 		session.Close()
 		return nil, err
 	}
 
-	chapters, err := newChapterRepository(db)
+	chapters, err := newChapterRepository(db, cfg.RestoreIDs)
 	if err != nil {
 		session.Close()
 		return nil, err
 	}
 
-	comments, err := newCommentRepository(db)
+	comments, err := newCommentRepository(db, cfg.RestoreIDs)
 	if err != nil {
 		session.Close()
 		return nil, err
 	}
 
-	keys, err := newKeyRepository(db)
+	keys, err := newKeyRepository(db, cfg.RestoreIDs)
 	if err != nil {
 		session.Close()
 		return nil, err
@@ -169,7 +171,9 @@ func Init(cfg config.Database) (*MongoDB, error) {
 		return nil, err
 	}
 
-	go posts.fixPositions(logs)
+	if !DisableFixes {
+		go posts.fixPositions(logs)
+	}
 
 	return &MongoDB{
 		session: session,
@@ -227,6 +231,29 @@ func (c *counter) With(category, name string) *counter {
 	}
 }
 
+func (c *counter) Bump(amount int) error {
+	id := c.category + "." + c.name
+
+	err := c.coll.Update(bson.M{
+		"_id":   id,
+		"value": bson.M{"$lt": amount},
+	}, bson.M{
+		"value": amount,
+	})
+	if err != nil {
+		if err == mgo.ErrNotFound {
+			return c.coll.Insert(bson.M{
+				"_id":   id,
+				"value": amount,
+			})
+		}
+
+		return err
+	}
+
+	return nil
+}
+
 func (c *counter) Increment(amount int) (int, error) {
 	type counterDoc struct {
 		ID    string `bson:"_id"`

database/mongodb/keys.go

@@ -10,10 +10,11 @@ import (
 )
 
 type keyRepository struct {
-	keys *mgo.Collection
+	restoreIDs bool
+	keys       *mgo.Collection
 }
 
-func newKeyRepository(db *mgo.Database) (repositories.KeyRepository, error) {
+func newKeyRepository(db *mgo.Database, restoreIDs bool) (repositories.KeyRepository, error) {
 	collection := db.C("auth.keys")
 
 	err := collection.EnsureIndexKey("user")
@@ -21,7 +22,10 @@ func newKeyRepository(db *mgo.Database) (repositories.KeyRepository, error) {
 		return nil, err
 	}
 
-	return &keyRepository{keys: collection}, nil
+	return &keyRepository{
+		keys:       collection,
+		restoreIDs: restoreIDs,
+	}, nil
 }
 
 func (r *keyRepository) Find(ctx context.Context, id string) (*models.Key, error) {
@@ -54,7 +58,9 @@ func (r *keyRepository) List(ctx context.Context, filter models.KeyFilter) ([]*m
 }
 
 func (r *keyRepository) Insert(ctx context.Context, key models.Key) (*models.Key, error) {
-	key.ID = generate.KeyID()
+	if !r.restoreIDs {
+		key.ID = generate.KeyID()
+	}
 
 	err := r.keys.Insert(&key)
 	if err != nil {

database/mongodb/logs.go

@@ -12,14 +12,15 @@ import (
 )
 
 type logRepository struct {
-	openMutex sync.Mutex
+	openMutex  sync.Mutex
+	restoreIds bool
 
 	logs           *mgo.Collection
 	posts          *mgo.Collection
 	shortIdCounter *counter
 }
 
-func newLogRepository(db *mgo.Database) (*logRepository, error) {
+func newLogRepository(db *mgo.Database, restoreIds bool) (*logRepository, error) {
 	logs := db.C("logbot3.logs")
 	posts := db.C("logbot3.posts")
 
@@ -54,6 +55,7 @@ func newLogRepository(db *mgo.Database) (*logRepository, error) {
 	}
 
 	return &logRepository{
+		restoreIds:     restoreIds,
 		logs:           logs,
 		posts:          posts,
 		shortIdCounter: newCounter(db, "auto_increment", "Log"),
@@ -123,26 +125,35 @@ func (r *logRepository) List(ctx context.Context, filter models.LogFilter) ([]*m
 }
 
 func (r *logRepository) Insert(ctx context.Context, log models.Log) (*models.Log, error) {
-	nextShortId, err := r.shortIdCounter.Increment(1)
-	if err != nil {
-		return nil, err
-	}
+	if !r.restoreIds || log.ID == "" || log.ShortID == "" {
+		nextShortId, err := r.shortIdCounter.Increment(1)
+		if err != nil {
+			return nil, err
+		}
 
-	log.ID = generate.LogID(log)
-	log.ShortID = "L" + strconv.Itoa(nextShortId)
+		log.ID = generate.LogID(log)
+		log.ShortID = "L" + strconv.Itoa(nextShortId)
+	} else {
+		n, err := strconv.Atoi(log.ShortID[1:])
+		if err != nil {
+			return nil, err
+		}
+
+		_ = r.shortIdCounter.Bump(n)
+	}
 
 	if log.Open {
 		// There can be only one open log in the same channel.
 		r.openMutex.Lock()
 		defer r.openMutex.Unlock()
 
-		_, err = r.logs.UpdateAll(bson.M{"channel": log.ChannelName, "open": true}, bson.M{"$set": bson.M{"open": false}})
+		_, err := r.logs.UpdateAll(bson.M{"channel": log.ChannelName, "open": true}, bson.M{"$set": bson.M{"open": false}})
 		if err != nil {
 			return nil, errors.New("Cannot close other logs: " + err.Error())
 		}
 	}
 
-	err = r.logs.Insert(&log)
+	err := r.logs.Insert(&log)
 	if err != nil {
 		return nil, err
 	}

database/mongodb/posts.go

@@ -2,6 +2,7 @@ package mongodb
 
 import (
 	"context"
+	"errors"
 	"git.aiterp.net/rpdata/api/internal/generate"
 	"git.aiterp.net/rpdata/api/models"
 	"git.aiterp.net/rpdata/api/repositories"
@@ -14,13 +15,14 @@ import (
 )
 
 type postRepository struct {
-	logs  *mgo.Collection
-	posts *mgo.Collection
+	restoreIDs bool
+	logs       *mgo.Collection
+	posts      *mgo.Collection
 
 	orderMutex sync.Mutex
 }
 
-func newPostRepository(db *mgo.Database) (*postRepository, error) {
+func newPostRepository(db *mgo.Database, restoreIDs bool) (*postRepository, error) {
 	posts := db.C("logbot3.posts")
 
 	err := posts.EnsureIndexKey("logId")
@@ -48,8 +50,9 @@ func newPostRepository(db *mgo.Database) (*postRepository, error) {
 	}
 
 	return &postRepository{
-		posts: posts,
-		logs:  db.C("logbot3.logs"),
+		restoreIDs: restoreIDs,
+		posts:      posts,
+		logs:       db.C("logbot3.logs"),
 	}, nil
 }
 
@@ -91,7 +94,12 @@ func (r *postRepository) List(ctx context.Context, filter models.PostFilter) ([]
 	}
 
 	posts := make([]*models.Post, 0, 32)
-	err := r.posts.Find(query).Sort("-logId", "position").Limit(filter.Limit).All(&posts)
+	var err error
+	if filter.LogID != nil {
+		err = r.posts.Find(query).Sort("position").Limit(filter.Limit).All(&posts)
+	} else {
+		err = r.posts.Find(query).Limit(filter.Limit).All(&posts)
+	}
 	if err != nil {
 		if err == mgo.ErrNotFound {
 			return []*models.Post{}, nil
@@ -113,7 +121,14 @@ func (r *postRepository) Insert(ctx context.Context, post models.Post) (*models.
 		return nil, err
 	}
 
-	post.ID = generate.PostID()
+	if !r.restoreIDs {
+		post.ID = generate.PostID()
+	} else {
+		if len(post.ID) != len(generate.PostID()) && strings.HasPrefix(post.ID, "P") {
+			return nil, errors.New("invalid story id")
+		}
+	}
+
 	post.Position = lastPost.Position + 1 // Position 1 is first position, so this is safe.
 
 	err = r.posts.Insert(post)
@@ -135,7 +150,9 @@ func (r *postRepository) InsertMany(ctx context.Context, posts ...*models.Post)
 			return nil, repositories.ErrParentMismatch
 		}
 
-		post.ID = generate.PostID()
+		if !r.restoreIDs || post.ID == "" {
+			post.ID = generate.PostID()
+		}
 	}
 
 	r.orderMutex.Lock()

database/mongodb/stories.go

@@ -2,21 +2,24 @@ package mongodb
 
 import (
 	"context"
+	"errors"
 	"git.aiterp.net/rpdata/api/internal/generate"
 	"git.aiterp.net/rpdata/api/models"
 	"git.aiterp.net/rpdata/api/repositories"
 	"github.com/globalsign/mgo"
 	"github.com/globalsign/mgo/bson"
 	"log"
+	"strings"
 )
 
 type storyRepository struct {
-	stories  *mgo.Collection
-	chapters *mgo.Collection
-	comments *mgo.Collection
+	stories    *mgo.Collection
+	chapters   *mgo.Collection
+	comments   *mgo.Collection
+	restoreIDs bool
 }
 
-func newStoryRepository(db *mgo.Database) (repositories.StoryRepository, error) {
+func newStoryRepository(db *mgo.Database, restoreIDs bool) (repositories.StoryRepository, error) {
 	collection := db.C("story.stories")
 
 	err := collection.EnsureIndexKey("tags")
@@ -41,9 +44,10 @@ func newStoryRepository(db *mgo.Database) (repositories.StoryRepository, error)
 	}
 
 	return &storyRepository{
-		stories:  collection,
-		chapters: db.C("story.chapters"),
-		comments: db.C("story.comments"),
+		stories:    collection,
+		restoreIDs: restoreIDs,
+		chapters:   db.C("story.chapters"),
+		comments:   db.C("story.comments"),
 	}, nil
 }
 
@@ -105,7 +109,13 @@ func (r *storyRepository) List(ctx context.Context, filter models.StoryFilter) (
 }
 
 func (r *storyRepository) Insert(ctx context.Context, story models.Story) (*models.Story, error) {
-	story.ID = generate.StoryID()
+	if !r.restoreIDs {
+		story.ID = generate.StoryID()
+	} else {
+		if len(story.ID) != len(generate.StoryID()) && strings.HasPrefix(story.ID, "S") {
+			return nil, errors.New("invalid story id")
+		}
+	}
 
 	err := r.stories.Insert(story)
 	if err != nil {
@@ -145,6 +155,10 @@ func (r *storyRepository) Update(ctx context.Context, story models.Story, update
 		updateBson["listed"] = *update.Listed
 		story.Listed = *update.Listed
 	}
+	if update.SortByFictionalDate != nil {
+		updateBson["sortByFictionalDate"] = *update.SortByFictionalDate
+		story.SortByFictionalDate = *update.SortByFictionalDate
+	}
 
 	err := r.stories.UpdateId(story.ID, bson.M{"$set": updateBson})
 	if err != nil {

database/mongodb/users.go

@@ -5,6 +5,7 @@ import (
 	"git.aiterp.net/rpdata/api/models"
 	"git.aiterp.net/rpdata/api/repositories"
 	"github.com/globalsign/mgo"
+	"github.com/globalsign/mgo/bson"
 )
 
 type userRepository struct {
@@ -31,6 +32,16 @@ func (r *userRepository) Find(ctx context.Context, id string) (*models.User, err
 	return user, nil
 }
 
+func (r *userRepository) List(ctx context.Context) ([]*models.User, error) {
+	users := make([]*models.User, 0, 8)
+	err := r.users.Find(bson.M{}).All(&users)
+	if err != nil && err != mgo.ErrNotFound {
+		return nil, err
+	}
+
+	return users, nil
+}
+
 func (r *userRepository) Insert(ctx context.Context, user models.User) (*models.User, error) {
 	err := r.users.Insert(user)
 	if err != nil {

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/globalsign/mgo v0.0.0-20180403085842-f76e4f9da92e
 	github.com/go-ini/ini v1.35.0 // indirect
 	github.com/golang/protobuf v1.3.1 // indirect
+	github.com/golang/snappy v0.0.2 // indirect
 	github.com/google/go-cmp v0.3.0 // indirect
 	github.com/gorilla/websocket v1.4.0 // indirect
 	github.com/h2non/filetype v1.0.8
@@ -23,6 +24,8 @@ require (
 	github.com/tidwall/pretty v1.0.0 // indirect
 	github.com/vektah/dataloaden v0.3.0
 	github.com/vektah/gqlparser v1.1.2
+	github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c // indirect
+	github.com/xdg/stringprep v1.0.0 // indirect
 	go.mongodb.org/mongo-driver v1.0.3
 	golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 // indirect
 	golang.org/x/net v0.0.0-20190514140710-3ec191127204 // indirect

go.sum

@@ -29,6 +29,8 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/snappy v0.0.2 h1:aeE13tS0IiQgFjYdoL8qN3K1N2bXXtI6Vi51/y7BpMw=
+github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
@@ -104,6 +106,10 @@ github.com/vektah/dataloaden v0.3.0 h1:ZfVN2QD6swgvp+tDqdH/OIT/wu3Dhu0cus0k5gIZS
 github.com/vektah/dataloaden v0.3.0/go.mod h1:/HUdMve7rvxZma+2ZELQeNh88+003LL7Pf/CZ089j8U=
 github.com/vektah/gqlparser v1.1.2 h1:ZsyLGn7/7jDNI+y4SEhI4yAxRChlv15pUHMjijT+e68=
 github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
+github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk=
+github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
+github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0=
+github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
 go.mongodb.org/mongo-driver v1.0.3 h1:GKoji1ld3tw2aC+GX1wbr/J2fX13yNacEYoJ8Nhr0yU=
 go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I=

graph2/resolvers/log.go

@@ -55,7 +55,12 @@ func (r *mutationResolver) ImportLog(ctx context.Context, input graphcore.LogImp
 		tz = parsedTZ
 	}
 
-	return r.s.Logs.Import(ctx, input.Importer, date, tz, input.ChannelName, input.Data)
+	sessionThreshold := time.Hour * 12
+	if input.SessionThresholdMs != nil {
+		sessionThreshold = time.Duration(*input.SessionThresholdMs) * time.Millisecond
+	}
+
+	return r.s.Logs.Import(ctx, input.Importer, date, tz, input.ChannelName, sessionThreshold, input.Data)
 }
 
 func (r *mutationResolver) SplitLog(ctx context.Context, input graphcore.LogSplitInput) (*models.Log, error) {

graph2/resolvers/story.go

@@ -36,7 +36,9 @@ func (r *mutationResolver) AddStory(ctx context.Context, input graphcore.StoryAd
 		fictionalDate = *input.FictionalDate
 	}
 
-	return r.s.Stories.CreateStory(ctx, input.Name, input.Author, input.Category, listed, open, tags, time.Now(), fictionalDate)
+	sortByFictionalDate := input.SortByFictionalDate != nil && *input.SortByFictionalDate
+
+	return r.s.Stories.CreateStory(ctx, input.Name, input.Author, input.Category, listed, open, tags, time.Now(), fictionalDate, sortByFictionalDate)
 }
 
 func (r *mutationResolver) AddStoryTag(ctx context.Context, input graphcore.StoryTagAddInput) (*models.Story, error) {
@@ -63,7 +65,7 @@ func (r *mutationResolver) EditStory(ctx context.Context, input graphcore.StoryE
 		return nil, err
 	}
 
-	return r.s.Stories.EditStory(ctx, story, input.Name, input.Category, input.Listed, input.Open, input.FictionalDate)
+	return r.s.Stories.EditStory(ctx, story, input.Name, input.Category, input.Listed, input.Open, input.FictionalDate, input.SortByFictionalDate)
 }
 
 func (r *mutationResolver) RemoveStory(ctx context.Context, input graphcore.StoryRemoveInput) (*models.Story, error) {

graph2/schema/types/Log.gql

@@ -132,6 +132,9 @@ input LogImportInput {
   "The date of the log, if not provided in the log body."
   date: Time
 
+  "Session threwhold"
+  sessionThresholdMs: Int
+
   "The log body itself."
   data: String!
 }
@@ -171,4 +174,9 @@ enum LogImporter {
   Forum log: This importer parses the format on the forum. The displayed log, not the post source.
   """
   ForumLog
+
+  """
+  IRCCloud: This importer parses irccloud exports.
+  """
+  IrcCloud
 }

graph2/schema/types/Story.gql

@@ -33,6 +33,9 @@ type Story {
 
   # The date of the last major update to the story. This being bumped means a new chapter has been added.
   updatedDate: Time!
+
+  # Whether to sort by fictional date.
+  sortByFictionalDate: Boolean!
 }
 
 # Filter for stories query.
@@ -86,6 +89,9 @@ input StoryAddInput {
 
   # Set the fictional date of the story.
   fictionalDate: Time
+
+  # Whether to sort by fictional date.
+  sortByFictionalDate: Boolean
 }
 
 # Input for the editStory mutation
@@ -115,6 +121,9 @@ input StoryEditInput {
 
   # Clear the fictional date of the story.
   clearFictionalDate: Boolean
+
+  # Whether to sort by fictional date.
+  sortByFictionalDate: Boolean
 }
 
 # Input for the addStoryTag mutation

internal/config/config.go

@@ -35,13 +35,14 @@ type Space struct {
 
 // Database is configuration for spaces.
 type Database struct {
-	Driver    string `json:"driver" yaml:"driver"`
-	Host      string `json:"host" yaml:"host"`
-	Port      int    `json:"port" yaml:"port"`
-	Db        string `json:"db" yaml:"db"`
-	Username  string `json:"username" yaml:"username"`
-	Password  string `json:"password" yaml:"password"`
-	Mechanism string `json:"mechanism" yaml:"mechanism"`
+	Driver     string `json:"driver" yaml:"driver"`
+	Host       string `json:"host" yaml:"host"`
+	Port       int    `json:"port" yaml:"port"`
+	Db         string `json:"db" yaml:"db"`
+	Username   string `json:"username" yaml:"username"`
+	Password   string `json:"password" yaml:"password"`
+	Mechanism  string `json:"mechanism" yaml:"mechanism"`
+	RestoreIDs bool   `json:"restoreIDs" yaml:"restoreIDs"`
 }
 
 // Wiki is the wiki stuff.

models/log-importer.go

@@ -14,12 +14,14 @@ const (
 	LogImporterMircLike LogImporter = "MircLike"
 	// LogImporterForumLog is a value of LogImporter
 	LogImporterForumLog LogImporter = "ForumLog"
+	// LogImporterForumLog is a value of LogImporter
+	LogImporterIrcCloud LogImporter = "IrcCloud"
 )
 
 // IsValid returns true if the underlying string is one of the correct values.
 func (e LogImporter) IsValid() bool {
 	switch e {
-	case LogImporterForumLog, LogImporterMircLike:
+	case LogImporterForumLog, LogImporterMircLike, LogImporterIrcCloud:
 		return true
 	}
 	return false

models/story.go

@@ -5,16 +5,17 @@ import "time"
 // A Story is user content that does not have a wiki-suitable format. Documents, new stories, short stories, and so on.
 // The story model is a container for multiple chapters this time, in contrast to the previous version.
 type Story struct {
-	ID            string        `bson:"_id"`
-	Author        string        `bson:"author"`
-	Name          string        `bson:"name"`
-	Category      StoryCategory `bson:"category"`
-	Open          bool          `bson:"open"`
-	Listed        bool          `bson:"listed"`
-	Tags          []Tag         `bson:"tags"`
-	CreatedDate   time.Time     `bson:"createdDate"`
-	FictionalDate time.Time     `bson:"fictionalDate,omitempty"`
-	UpdatedDate   time.Time     `bson:"updatedDate"`
+	ID                  string        `bson:"_id"`
+	Author              string        `bson:"author"`
+	Name                string        `bson:"name"`
+	Category            StoryCategory `bson:"category"`
+	Open                bool          `bson:"open"`
+	Listed              bool          `bson:"listed"`
+	Tags                []Tag         `bson:"tags"`
+	CreatedDate         time.Time     `bson:"createdDate"`
+	FictionalDate       time.Time     `bson:"fictionalDate,omitempty"`
+	UpdatedDate         time.Time     `bson:"updatedDate"`
+	SortByFictionalDate bool          `bson:"sortByFictionalDate"`
 }
 
 // IsChangeObject is an interface implementation to identify it as a valid
@@ -35,11 +36,12 @@ type StoryFilter struct {
 }
 
 type StoryUpdate struct {
-	Name          *string
-	Category      *StoryCategory
-	Author        *string
-	Open          *bool
-	Listed        *bool
-	FictionalDate *time.Time
-	UpdatedDate   *time.Time
+	Name                *string
+	Category            *StoryCategory
+	Author              *string
+	Open                *bool
+	Listed              *bool
+	FictionalDate       *time.Time
+	UpdatedDate         *time.Time
+	SortByFictionalDate *bool
 }

repositories/user.go

@@ -7,5 +7,6 @@ import (
 
 type UserRepository interface {
 	Find(ctx context.Context, id string) (*models.User, error)
+	List(ctx context.Context) ([]*models.User, error)
 	Insert(ctx context.Context, user models.User) (*models.User, error)
 }

services/auth.go

@@ -125,6 +125,11 @@ func (s *AuthService) CheckPermission(ctx context.Context, op string, obj interf
 	case *models.Post:
 		authorized = token.Permitted("post." + op)
 	case *models.Story:
+		if op == "tag" && v.Open {
+			authorized = true
+			break
+		}
+
 		authorized = token.PermittedUser(v.Author, "member", "story."+op)
 	case *models.User:
 		authorized = token.Permitted("user." + op)

services/characters.go

@@ -7,8 +7,11 @@ import (
 	"git.aiterp.net/rpdata/api/models/changekeys"
 	"git.aiterp.net/rpdata/api/repositories"
 	"git.aiterp.net/rpdata/api/services/loaders"
+	"log"
 	"sort"
 	"strings"
+	"sync/atomic"
+	"time"
 )
 
 type CharacterService struct {
@@ -16,6 +19,9 @@ type CharacterService struct {
 	loader        *loaders.CharacterLoader
 	changeService *ChangeService
 	authService   *AuthService
+	logService    *LogService
+
+	refreshSoftLock uint32
 }
 
 // Find uses the loader to find the character by the ID.
@@ -61,6 +67,8 @@ func (s *CharacterService) List(ctx context.Context, filter models.CharacterFilt
 		return characters, nil
 	}
 
+	go s.refreshLogs()
+
 	return s.characters.List(ctx, filter)
 }
 
@@ -108,6 +116,8 @@ func (s *CharacterService) Create(ctx context.Context, nick, name, shortName, au
 
 	s.changeService.Submit(ctx, "Character", "add", true, changekeys.Listed(character), character)
 
+	go s.refreshLogs()
+
 	return character, nil
 }
 
@@ -160,6 +170,8 @@ func (s *CharacterService) AddNick(ctx context.Context, id string, nick string)
 
 	s.changeService.Submit(ctx, "Character", "edit", true, changekeys.Listed(character), character)
 
+	go s.refreshLogs()
+
 	return character, nil
 }
 
@@ -184,6 +196,8 @@ func (s *CharacterService) RemoveNick(ctx context.Context, id string, nick strin
 
 	s.changeService.Submit(ctx, "Character", "edit", true, changekeys.Listed(character), character)
 
+	go s.refreshLogs()
+
 	return character, nil
 }
 
@@ -207,5 +221,21 @@ func (s *CharacterService) Delete(ctx context.Context, id string) (*models.Chara
 
 	s.changeService.Submit(ctx, "Character", "remove", true, changekeys.Listed(character), character)
 
+	go s.refreshLogs()
+
 	return character, nil
 }
+
+func (s *CharacterService) refreshLogs() {
+	if !atomic.CompareAndSwapUint32(&s.refreshSoftLock, 0, 1) {
+		return
+	}
+	time.Sleep(time.Second * 60)
+	atomic.StoreUint32(&s.refreshSoftLock, 0)
+
+	err := s.logService.RefreshAllLogCharacters(context.Background())
+	if err != nil {
+		log.Println("Failed to refersh log characters:", err)
+		return
+	}
+}

services/logs.go

@@ -112,7 +112,7 @@ func (s *LogService) Create(ctx context.Context, title, description, channelName
 }
 
 // Import creates new logs from common formats.
-func (s *LogService) Import(ctx context.Context, importer models.LogImporter, date time.Time, tz *time.Location, channelName string, data string) ([]*models.Log, error) {
+func (s *LogService) Import(ctx context.Context, importer models.LogImporter, date time.Time, tz *time.Location, channelName string, sessionThreshold time.Duration, data string) ([]*models.Log, error) {
 	if err := s.authService.CheckPermission(ctx, "add", &models.Log{}); err != nil {
 		return nil, err
 	}
@@ -145,26 +145,33 @@ func (s *LogService) Import(ctx context.Context, importer models.LogImporter, da
 			parsed.Log.EventName = eventName
 			parsed.Log.ChannelName = channelName
 
-			log, err := s.logs.Insert(ctx, parsed.Log)
+			newLog, err := s.logs.Insert(ctx, parsed.Log)
 			if err != nil {
 				return nil, err
 			}
 
 			for _, post := range parsed.Posts {
-				post.LogID = log.ShortID
+				post.LogID = newLog.ShortID
 			}
 
 			posts, err := s.posts.InsertMany(ctx, parsed.Posts...)
 			if err != nil {
-				_ = s.logs.Delete(ctx, *log)
+				_ = s.logs.Delete(ctx, *newLog)
 
 				return nil, err
 			}
 
-			s.changeService.Submit(ctx, models.ChangeModelLog, "add", true, changekeys.Listed(log), log)
-			s.changeService.Submit(ctx, models.ChangeModelPost, "add", true, changekeys.Listed(log, posts), log, posts)
+			s.changeService.Submit(ctx, models.ChangeModelLog, "add", true, changekeys.Listed(newLog), newLog)
+			s.changeService.Submit(ctx, models.ChangeModelPost, "add", true, changekeys.Listed(newLog, posts), newLog, posts)
 
-			results = append(results, log)
+			refreshedLog, err := s.refreshLogCharacters(ctx, *newLog, nil, false)
+			if err != nil {
+				log.Printf("Failed to update characters in newLog %s: %s", newLog.ID, err)
+			} else {
+				newLog = refreshedLog
+			}
+
+			results = append(results, newLog)
 		}
 	case models.LogImporterForumLog:
 		{
@@ -174,28 +181,73 @@ func (s *LogService) Import(ctx context.Context, importer models.LogImporter, da
 			}
 
 			for _, parsed := range parseResults {
-				log, err := s.logs.Insert(ctx, parsed.Log)
-				if err != nil {
-					return nil, err
-				}
 				parsed.Log.EventName = eventName
 				parsed.Log.ChannelName = channelName
 
+				newLog, err := s.logs.Insert(ctx, parsed.Log)
+				if err != nil {
+					return nil, err
+				}
+
 				for _, post := range parsed.Posts {
-					post.LogID = log.ShortID
+					post.LogID = newLog.ShortID
 				}
 
 				posts, err := s.posts.InsertMany(ctx, parsed.Posts...)
 				if err != nil {
-					_ = s.logs.Delete(ctx, *log)
+					_ = s.logs.Delete(ctx, *newLog)
 
 					return nil, err
 				}
 
-				s.changeService.Submit(ctx, models.ChangeModelLog, "add", true, changekeys.Listed(log), log)
-				s.changeService.Submit(ctx, models.ChangeModelPost, "add", true, changekeys.Listed(log, posts), log, posts)
+				s.changeService.Submit(ctx, models.ChangeModelLog, "add", true, changekeys.Listed(newLog), newLog)
+				s.changeService.Submit(ctx, models.ChangeModelPost, "add", true, changekeys.Listed(newLog, posts), newLog, posts)
+
+				results = append(results, newLog)
+			}
+
+		}
+	case models.LogImporterIrcCloud:
+		{
+			parseResults, err := parsers.IRCCloudLogs(data, tz, sessionThreshold)
+			if err != nil {
+				return nil, err
+			}
+
+			for _, parsed := range parseResults {
+				parsed.Log.EventName = eventName
+				parsed.Log.ChannelName = channelName
+
+				newLog, err := s.logs.Insert(ctx, parsed.Log)
+				if err != nil {
+					return nil, err
+				}
+
+				for _, post := range parsed.Posts {
+					post.LogID = newLog.ShortID
+				}
+
+				posts, err := s.posts.InsertMany(ctx, parsed.Posts...)
+				if err != nil {
+					_ = s.logs.Delete(ctx, *newLog)
+
+					return results, err
+				}
+
+				refreshedLog, err := s.refreshLogCharacters(ctx, *newLog, nil, false)
+				if err != nil {
+					log.Printf("Failed to update characters in newLog %s: %s", newLog.ID, err)
+				} else {
+					newLog = refreshedLog
+				}
+
+				s.changeService.Submit(ctx, models.ChangeModelLog, "add", true, changekeys.Listed(newLog), newLog)
+				s.changeService.Submit(ctx, models.ChangeModelPost, "add", true, changekeys.Listed(newLog, posts), newLog, posts)
+
+				results = append(results, newLog)
 			}
 		}
+
 	default:
 		{
 			return nil, errors.New("Invalid importer: " + importer.String())

services/parsers/irccloud.go

@@ -0,0 +1,161 @@
+package parsers
+
+import (
+	"errors"
+	"fmt"
+	"git.aiterp.net/rpdata/api/models"
+	"strings"
+	"time"
+)
+
+var ErrSkip = errors.New("parsers: skip this post")
+
+func IRCCloudLogs(data string, location *time.Location, threshold time.Duration) ([]ParsedLog, error) {
+	lines := strings.Split(data, "\n")
+	pos := 0
+	results := make([]ParsedLog, 0, 8)
+	for pos < len(lines) {
+		log, n, err := IRCCloudLog(lines[pos:], location, threshold)
+		if err != nil {
+			if err == ErrEmptyLog {
+				pos += n
+				continue
+			}
+
+			return nil, err
+		}
+
+		pos += n
+		results = append(results, *log)
+	}
+
+	return results, nil
+}
+
+// IRCCloudLog parses the log and returns the things that can be gleamed from them.
+func IRCCloudLog(lines []string, location *time.Location, threshold time.Duration) (*ParsedLog, int, error) {
+	posts := make([]*models.Post, 0, len(lines))
+	prev := (*models.Post)(nil)
+	amount := 0
+
+	for _, line := range lines {
+		line = strings.Trim(line, "\r\t  ")
+		if len(line) < 1 {
+			amount += 1
+			continue
+		}
+
+		post, err := IRCCloudPost(line, location)
+		if err == ErrSkip {
+			amount += 1
+			continue
+		} else if err != nil {
+			return nil, -1, err
+		}
+
+		if prev != nil {
+			if post.Time.Sub(prev.Time) >= threshold {
+				break
+			}
+
+			post.Position = prev.Position + 1
+		} else {
+			post.Position = 1
+		}
+
+		posts = append(posts, &post)
+		prev = &post
+		amount += 1
+	}
+
+	if len(posts) == 0 {
+		return nil, amount, ErrEmptyLog
+	}
+
+	log := models.Log{
+		Date: posts[0].Time,
+	}
+
+	return &ParsedLog{log, posts}, amount, nil
+}
+
+// IRCCloudPost parses a post from a mirc-like line. If the previous post is included (it can be empty), it will be used
+// to determine whether midnight has passed.
+func IRCCloudPost(line string, tz *time.Location) (models.Post, error) {
+	// Do basic validation
+	line = strings.Trim(line, "  \t\n\r")
+	if len(line) == 0 || !strings.HasPrefix(line, "[") {
+		return models.Post{}, &ParseError{
+			Line:    line,
+			Problem: "no timestamp",
+		}
+	}
+
+	// Parse timestamp
+	tsEndIndex := strings.IndexByte(line, ']')
+	if tsEndIndex == -1 || len(line) < tsEndIndex+5 {
+		return models.Post{}, &ParseError{
+			Line:    line,
+			Problem: "incomplete timestamp",
+		}
+	}
+	tsStr := line[1:tsEndIndex]
+
+	ts, err := time.ParseInLocation("2006-01-02 15:04:05", tsStr, tz)
+	if err != nil {
+		return models.Post{}, &ParseError{
+			Line:    line,
+			Problem: fmt.Sprintf("Could not parse date: %s", err.Error()),
+		}
+	}
+
+	if strings.HasPrefix(line[tsEndIndex+2:], "—") {
+		split := strings.SplitN(line[tsEndIndex+6:], " ", 2)
+		if len(split) == 1 {
+			return models.Post{}, &ParseError{
+				Line:    line,
+				Problem: "post is empty",
+			}
+		}
+
+		post := models.Post{
+			ID:    "UNASSIGNED",
+			LogID: "UNASSIGNED",
+			Time:  ts,
+			Kind:  "action",
+			Nick:  strings.Trim(strings.TrimLeft(split[0], "+@!~"), "\u001F"),
+			Text:  split[1],
+		}
+
+		if strings.HasPrefix(post.Nick, "=") {
+			post.Kind = "scene"
+		}
+
+		return post, nil
+	} else if line[tsEndIndex+2] == '<' {
+		split := strings.SplitN(line[tsEndIndex+2:], " ", 2)
+		if len(split) == 1 {
+			return models.Post{}, &ParseError{
+				Line:    line,
+				Problem: "post is empty",
+			}
+		}
+
+		post := models.Post{
+			ID:    "UNASSIGNED",
+			LogID: "UNASSIGNED",
+			Time:  ts,
+			Kind:  "text",
+			Nick:  strings.Trim(strings.TrimLeft(split[0][1:len(split[0])-1], "+@!~"), "\u001F"),
+			Text:  split[1],
+		}
+
+		if strings.HasPrefix(post.Nick, "=") {
+			post.Kind = "scene"
+		}
+
+		return post, nil
+	} else {
+		return models.Post{}, ErrSkip
+	}
+}

services/parsers/irccloud_test.go

@@ -0,0 +1,154 @@
+package parsers
+
+import (
+	"fmt"
+	"github.com/stretchr/testify/assert"
+	"testing"
+	"time"
+)
+
+func TestIRCCloudPost(t *testing.T) {
+	table := []struct {
+		Input   string
+		TS      string
+		Kind    string
+		Nick    string
+		Text    string
+		Skipped bool
+	}{
+		{
+			"[2016-07-18 12:57:17] — \u001FCharacter_Name\u001F does things.",
+			"2016-07-18 12:57:17", "action", "Character_Name", "does things.",
+			false,
+		},
+		{
+			"[2016-07-18 13:04:06] — Character keeps their mouth shut again for the time beings.",
+			"2016-07-18 13:04:06", "action", "Character", "keeps their mouth shut again for the time beings.",
+			false,
+		},
+		{
+			"[2016-07-18 12:34:42] <Victoria_Steels> \"I can keep it professional.\"",
+			"2016-07-18 12:34:42", "text", "Victoria_Steels", "\"I can keep it professional.\"",
+			false,
+		},
+		{
+			"[2016-07-18 10:41:10] * UserZzz → User",
+			"", "", "", "",
+			true,
+		},
+		{
+			"[2016-07-18 12:53:55] → Someone joined",
+			"", "", "", "",
+			true,
+		},
+		{
+			"[2016-07-18 22:14:15] ⇐ SomeoneElse quit (s@example.com): Ping timeout: 547 seconds",
+			"", "", "", "",
+			true,
+		},
+	}
+
+	for i, row := range table {
+		t.Run(fmt.Sprintf("Row_%d", i), func(t *testing.T) {
+			post, err := IRCCloudPost(row.Input, time.UTC)
+			if err != nil && err != ErrSkip {
+				t.Fatal("Could not parse post:", err)
+			}
+
+			if row.Skipped {
+				if err == ErrSkip {
+					return
+				}
+
+				t.Fatal("Row should be skipped")
+			}
+			if !row.Skipped && err == ErrSkip {
+				t.Fatal("Row should not be skipped, but is")
+			}
+
+			assert.Equal(t, row.TS, post.Time.Format("2006-01-02 15:04:05"), "Timestamps should match.")
+			assert.Equal(t, row.Kind, post.Kind, "Kinds should match.")
+			assert.Equal(t, row.Nick, post.Nick, "Kinds should match.")
+			assert.Equal(t, row.Text, post.Text, "Kinds should match.")
+		})
+	}
+}
+
+func TestIRCCloudLog(t *testing.T) {
+	type logLine struct {
+		TS   string
+		Kind string
+		Nick string
+		Text string
+	}
+
+	logs := [][]logLine{
+		{
+			{"2011-09-23 15:37:16", "action", "Jason_Wolfe", "sheds a bit of the tension he was harboring as Markus extends his hand."},
+			{"2011-09-23 15:41:15", "action", "Markus_Vasquez", "gives another nod, but there's no smile from him."},
+			{"2011-09-23 15:47:22", "action", "Jason_Wolfe", "fights the temptation to shove his hand back in his pocket as he notices Markus."},
+			{"2011-09-23 15:47:22", "action", "Jason_Wolfe", "warmer to wear."},
+			{"2011-09-23 15:47:32", "text", "Dante", "((you're*))"},
+			{"2011-09-23 15:48:45", "action", "Markus_Vasquez", "grunts as they walk toward the door, shaking his head slightly."},
+		},
+		{
+			{"2011-09-25 21:03:55", "scene", "=Scene=", "It's early evening on the 11th of November, and the weather is clear."},
+			{"2011-09-25 21:03:55", "scene", "=Scene=", "café on the eastern edge of town."},
+			{"2011-09-25 21:08:33", "action", "Sofia_Tennhausen", "is sat on a small table outside the cafe, a cup of lukewarm coffee in her hand."},
+		},
+	}
+
+	results, err := IRCCloudLogs(testLog, time.UTC, time.Hour*6)
+	if err != nil {
+		t.Fatal("Parse", err)
+	}
+
+	if len(results) != len(logs) {
+		t.Fatal("Two logs expected, got", len(logs))
+	}
+
+	for i, log := range logs {
+		result := results[i]
+
+		if len(result.Posts) != len(log) {
+			t.Error(len(log), "posts expected in log", i, "but got", len(result.Posts))
+			for _, post := range result.Posts {
+				t.Log(*post)
+			}
+
+			return
+		}
+
+		for j, post := range result.Posts {
+			t.Run(fmt.Sprintf("Log_%d_Post_%d", i, j), func(t *testing.T) {
+				assert.Equal(t, log[j].TS, post.Time.Format("2006-01-02 15:04:05"), "Timestamps should match.")
+				assert.Equal(t, log[j].Kind, post.Kind, "Kinds should match.")
+				assert.Equal(t, log[j].Nick, post.Nick, "Kinds should match.")
+				assert.Equal(t, log[j].Text, post.Text, "Kinds should match.")
+			})
+		}
+
+	}
+}
+
+var testLog = `
+[2011-09-23 15:37:16] — Jason_Wolfe sheds a bit of the tension he was harboring as Markus extends his hand.
+[2011-09-23 15:41:15] — Markus_Vasquez gives another nod, but there's no smile from him.
+
+
+[2011-09-23 15:47:22] — Jason_Wolfe fights the temptation to shove his hand back in his pocket as he notices Markus.
+[2011-09-23 15:47:22] — Jason_Wolfe warmer to wear.
+[2011-09-23 15:47:32] <Dante> ((you're*))
+[2011-09-23 15:48:45] — Markus_Vasquez grunts as they walk toward the door, shaking his head slightly.
+[2011-09-23 18:00:28] ⇐ Tyranniac quit (Tyranniac@example.com): Ping timeout: 260 seconds
+[2011-09-23 18:58:39] → Tyranniac joined (Tyranniac@example.com)
+[2011-09-23 18:58:39] * ChanServ set +v Tyranniac
+[2011-09-24 01:45:21] → Hobo joined (Hobo@example.com)
+[2011-09-24 11:37:07] ⇐ Hobo quit (Hobo@example.com): 
+[2011-09-24 21:01:34] ⇐ Tyranniac quit (Tyranniac@example.com): Ping timeout: 264 seconds
+[2011-09-24 23:00:32] → Tyranniac joined (Tyranniac@example.com)
+[2011-09-25 20:54:43] * Bowe → Sofia_Tennhausen
+[2011-09-25 21:03:55] <=Scene=> It's early evening on the 11th of November, and the weather is clear.
+[2011-09-25 21:03:55] <=Scene=> café on the eastern edge of town.
+[2011-09-25 21:08:33] — Sofia_Tennhausen is sat on a small table outside the cafe, a cup of lukewarm coffee in her hand.
+`

services/parsers/mirclike.go

@@ -2,14 +2,26 @@ package parsers
 
 import (
 	"errors"
+	"fmt"
 	"git.aiterp.net/rpdata/api/models"
 	"strconv"
 	"strings"
 	"time"
 )
 
-// ErrNotPost is returned by parsePost if the line is empty or not a post.
-var ErrNotPost = errors.New("not a post")
+type ParseError struct {
+	Line    string
+	Problem string
+}
+
+func (e *ParseError) Error() string {
+	return fmt.Sprintf("Unrecognized post: %s (error: %s)", e.Line, e.Problem)
+}
+
+func IsParseError(err error) bool {
+	_, ok := err.(*ParseError)
+	return ok
+}
 
 // ErrEmptyLog is returned by ParseLog if there are no (valid) posts in the log.
 var ErrEmptyLog = errors.New("no valid posts found in log")
@@ -56,24 +68,36 @@ func MircPost(line string, date time.Time, prev models.Post) (models.Post, error
 	// Do basic validation
 	line = strings.Trim(line, "  \t\n\r")
 	if len(line) == 0 || !strings.HasPrefix(line, "[") {
-		return models.Post{}, ErrNotPost
+		return models.Post{}, &ParseError{
+			Line:    line,
+			Problem: "no timestamp",
+		}
 	}
 
 	// Parse timestamp
 	tsEndIndex := strings.IndexByte(line, ']')
 	if tsEndIndex == -1 || len(line) < tsEndIndex+5 {
-		return models.Post{}, ErrNotPost
+		return models.Post{}, &ParseError{
+			Line:    line,
+			Problem: "incomplete timestamp",
+		}
 	}
 	tsStr := line[1:tsEndIndex]
 	tsSplit := strings.Split(tsStr, ":")
 	tsUnits := make([]int, len(tsSplit))
 	if len(tsSplit) < 2 {
-		return models.Post{}, ErrNotPost
+		return models.Post{}, &ParseError{
+			Line:    line,
+			Problem: "invalid timestamp",
+		}
 	}
 	for i := range tsSplit {
 		n, err := strconv.Atoi(tsSplit[i])
 		if err != nil {
-			return models.Post{}, ErrNotPost
+			return models.Post{}, &ParseError{
+				Line:    line,
+				Problem: "invalid number in timestamp",
+			}
 		}
 
 		tsUnits[i] = n
@@ -94,7 +118,10 @@ func MircPost(line string, date time.Time, prev models.Post) (models.Post, error
 	if line[tsEndIndex+2] == '*' {
 		split := strings.SplitN(line[tsEndIndex+4:], " ", 2)
 		if len(split) == 1 {
-			return models.Post{}, ErrNotPost
+			return models.Post{}, &ParseError{
+				Line:    line,
+				Problem: "post is empty",
+			}
 		}
 
 		post := models.Post{
@@ -102,7 +129,7 @@ func MircPost(line string, date time.Time, prev models.Post) (models.Post, error
 			LogID:    "UNASSIGNED",
 			Time:     ts,
 			Kind:     "action",
-			Nick:     strings.TrimLeft(split[0], "+@!~"),
+			Nick:     strings.Trim(strings.TrimLeft(split[0], "+@!~\u001F"), "\u001F"),
 			Text:     split[1],
 			Position: prev.Position + 1,
 		}
@@ -115,7 +142,10 @@ func MircPost(line string, date time.Time, prev models.Post) (models.Post, error
 	} else if line[tsEndIndex+2] == '<' {
 		split := strings.SplitN(line[tsEndIndex+2:], " ", 2)
 		if len(split) == 1 {
-			return models.Post{}, ErrNotPost
+			return models.Post{}, &ParseError{
+				Line:    line,
+				Problem: "post is empty",
+			}
 		}
 
 		post := models.Post{
@@ -123,7 +153,7 @@ func MircPost(line string, date time.Time, prev models.Post) (models.Post, error
 			LogID:    "UNASSIGNED",
 			Time:     ts,
 			Kind:     "text",
-			Nick:     strings.TrimLeft(split[0][1:len(split[0])-1], "+@!~"),
+			Nick:     strings.Trim(strings.TrimLeft(split[0][1:len(split[0])-1], "+@!~"), "\u001F"),
 			Text:     split[1],
 			Position: prev.Position + 1,
 		}
@@ -134,6 +164,9 @@ func MircPost(line string, date time.Time, prev models.Post) (models.Post, error
 
 		return post, nil
 	} else {
-		return models.Post{}, ErrNotPost
+		return models.Post{}, &ParseError{
+			Line:    line,
+			Problem: "line is neither action nor text post",
+		}
 	}
 }

services/parsers/mirclike_test.go

@@ -30,6 +30,10 @@ func TestMircPost(t *testing.T) {
 			"[13:36:59] <Stuff> Things said.",
 			"13:36:59", "text", "Stuff", "Things said.",
 		},
+		{
+			"[13:36:59] <\u001FStuff\u001F> Things said.",
+			"13:36:59", "text", "Stuff", "Things said.",
+		},
 		{
 			"[23:59] <=Scene=> Scenery and such.",
 			"23:59:00", "scene", "=Scene=", "Scenery and such.",
@@ -58,29 +62,38 @@ func TestMircPost(t *testing.T) {
 func TestMircPostErrors(t *testing.T) {
 	table := []struct {
 		Input string
-		Err   error
+		Err   string
 	}{
-		{"[12:34] <Stuff> Things said.", nil},
-		{"[12:34] >Stuff> Things said.", parsers.ErrNotPost},
-		{"12:34] <Stuff> Things said.", parsers.ErrNotPost},
-		{"* Stuff Things said.", parsers.ErrNotPost},
-		{"", parsers.ErrNotPost},
-		{"[12:34 <Stuff> Things said.", parsers.ErrNotPost},
-		{"[TE:XT] <Stuff> Things said.", parsers.ErrNotPost},
-		{"[10] <Stuff> Things said.", parsers.ErrNotPost},
-		{"[12:34:56:789] <Stuff> Things said.", nil},
-		{"[12:34:56.789] <Stuff> Things said.", parsers.ErrNotPost},
-		{"[12:34] <Stuff>", parsers.ErrNotPost},
-		{"[12:34] * Stuff", parsers.ErrNotPost},
-		{"[12:34] =Scene=", parsers.ErrNotPost},
-		{"[12:34] <=Scene=>", parsers.ErrNotPost},
+		{"[12:34] <Stuff> Things said.", ""},
+		{"[12:34] >Stuff> Things said.", "line is neither action nor text post"},
+		{"12:34] <Stuff> Things said.", "no timestamp"},
+		{"* Stuff Things said.", "no timestamp"},
+		{"", "no timestamp"},
+		{"[12:34 <Stuff> Things said.", "incomplete timestamp"},
+		{"[TE:XT] <Stuff> Things said.", "invalid number in timestamp"},
+		{"[10] <Stuff> Things said.", "invalid timestamp"},
+		{"[12:34:56:789] <Stuff> Things said.", ""},
+		{"[12:34:56.789] <Stuff> Things said.", "invalid number in timestamp"},
+		{"[12:34] <Stuff>", "post is empty"},
+		{"[12:34] * Stuff", "post is empty"},
+		{"[12:34] =Scene=", "line is neither action nor text post"},
+		{"[12:34] <=Scene=>", "post is empty"},
 	}
 
 	for i, row := range table {
 		t.Run(fmt.Sprintf("Row_%d", i), func(t *testing.T) {
 			_, err := parsers.MircPost(row.Input, time.Now(), models.Post{})
 
-			assert.Equal(t, row.Err, err, "Error should match")
+			errProblem := ""
+			if err != nil {
+				if e2, ok := err.(*parsers.ParseError); ok {
+					errProblem = e2.Problem
+				} else {
+					errProblem = err.Error()
+				}
+			}
+
+			assert.Equal(t, row.Err, errProblem, "Error should match")
 		})
 	}
 }
@@ -160,7 +173,7 @@ func TestMircLogErrors(t *testing.T) {
 	_, err2 := parsers.MircLog("\n\n\n\n\n[14:57]* Stuff   \n\t\r\n", time.Time{}, true)
 
 	assert.Equal(t, parsers.ErrEmptyLog, err1)
-	assert.Equal(t, parsers.ErrNotPost, err2)
+	assert.True(t, parsers.IsParseError(err2))
 }
 
 func parseDate(t *testing.T, date string) time.Time {

services/services.go

@@ -67,5 +67,7 @@ func NewBundle(db database.Database, spaceClient *space.Client) *Bundle {
 		authService:      bundle.Auth,
 	}
 
+	bundle.Characters.logService = bundle.Logs
+
 	return bundle
 }

services/stories.go

@@ -7,6 +7,7 @@ import (
 	"git.aiterp.net/rpdata/api/models"
 	"git.aiterp.net/rpdata/api/models/changekeys"
 	"git.aiterp.net/rpdata/api/repositories"
+	"sort"
 	"time"
 )
 
@@ -37,14 +38,37 @@ func (s *StoryService) ListStories(ctx context.Context, filter models.StoryFilte
 }
 
 func (s *StoryService) ListChapters(ctx context.Context, story models.Story) ([]*models.Chapter, error) {
-	return s.chapters.List(ctx, models.ChapterFilter{StoryID: &story.ID, Limit: 0})
+	chapters, err := s.chapters.List(ctx, models.ChapterFilter{StoryID: &story.ID, Limit: 0})
+	if err != nil {
+		return nil, err
+	}
+
+	if story.SortByFictionalDate {
+		sort.Slice(chapters, func(i, j int) bool {
+			if !chapters[i].FictionalDate.IsZero() && !chapters[j].FictionalDate.IsZero() {
+				if chapters[i].FictionalDate.Equal(chapters[j].FictionalDate) {
+					return chapters[i].CreatedDate.Before(chapters[j].CreatedDate)
+				}
+
+				return chapters[i].FictionalDate.Before(chapters[j].FictionalDate)
+			} else if chapters[i].FictionalDate.IsZero() && !chapters[j].FictionalDate.IsZero() {
+				return false
+			} else if !chapters[i].FictionalDate.IsZero() && chapters[j].FictionalDate.IsZero() {
+				return true
+			} else {
+				return chapters[i].CreatedDate.Before(chapters[j].CreatedDate)
+			}
+		})
+	}
+
+	return chapters, nil
 }
 
 func (s *StoryService) ListComments(ctx context.Context, chapter models.Chapter, limit int) ([]*models.Comment, error) {
 	return s.comments.List(ctx, models.CommentFilter{ChapterID: &chapter.ID, Limit: limit})
 }
 
-func (s *StoryService) CreateStory(ctx context.Context, name string, author *string, category models.StoryCategory, listed, open bool, tags []models.Tag, createdDate, fictionalDate time.Time) (*models.Story, error) {
+func (s *StoryService) CreateStory(ctx context.Context, name string, author *string, category models.StoryCategory, listed, open bool, tags []models.Tag, createdDate, fictionalDate time.Time, sortByFictionalDate bool) (*models.Story, error) {
 	if author == nil {
 		token := s.authService.TokenFromContext(ctx)
 		if token == nil {
@@ -55,15 +79,16 @@ func (s *StoryService) CreateStory(ctx context.Context, name string, author *str
 	}
 
 	story := &models.Story{
-		Name:          name,
-		Author:        *author,
-		Category:      category,
-		Listed:        listed,
-		Open:          open,
-		Tags:          tags,
-		CreatedDate:   createdDate,
-		FictionalDate: fictionalDate,
-		UpdatedDate:   createdDate,
+		Name:                name,
+		Author:              *author,
+		Category:            category,
+		Listed:              listed,
+		Open:                open,
+		Tags:                tags,
+		CreatedDate:         createdDate,
+		FictionalDate:       fictionalDate,
+		UpdatedDate:         createdDate,
+		SortByFictionalDate: sortByFictionalDate,
 	}
 
 	if err := s.authService.CheckPermission(ctx, "add", story); err != nil {
@@ -180,7 +205,7 @@ func (s *StoryService) CreateComment(ctx context.Context, chapter models.Chapter
 	return comment, nil
 }
 
-func (s *StoryService) EditStory(ctx context.Context, story *models.Story, name *string, category *models.StoryCategory, listed, open *bool, fictionalDate *time.Time) (*models.Story, error) {
+func (s *StoryService) EditStory(ctx context.Context, story *models.Story, name *string, category *models.StoryCategory, listed, open *bool, fictionalDate *time.Time, sortByFictionalDate *bool) (*models.Story, error) {
 	if story == nil {
 		panic("StoryService.Edit called with nil story")
 	}
@@ -190,11 +215,12 @@ func (s *StoryService) EditStory(ctx context.Context, story *models.Story, name
 	}
 
 	story, err := s.stories.Update(ctx, *story, models.StoryUpdate{
-		Name:          name,
-		Open:          open,
-		Listed:        listed,
-		Category:      category,
-		FictionalDate: fictionalDate,
+		Name:                name,
+		Open:                open,
+		Listed:              listed,
+		Category:            category,
+		FictionalDate:       fictionalDate,
+		SortByFictionalDate: sortByFictionalDate,
 	})
 	if err != nil {
 		return nil, err
@@ -206,7 +232,7 @@ func (s *StoryService) EditStory(ctx context.Context, story *models.Story, name
 }
 
 func (s *StoryService) AddStoryTag(ctx context.Context, story models.Story, tag models.Tag) (*models.Story, error) {
-	if err := s.authService.CheckPermission(ctx, "edit", &story); err != nil {
+	if err := s.authService.CheckPermission(ctx, "tag", &story); err != nil {
 		return nil, err
 	}
 
@@ -223,7 +249,7 @@ func (s *StoryService) AddStoryTag(ctx context.Context, story models.Story, tag
 }
 
 func (s *StoryService) RemoveStoryTag(ctx context.Context, story models.Story, tag models.Tag) (*models.Story, error) {
-	if err := s.authService.CheckPermission(ctx, "edit", &story); err != nil {
+	if err := s.authService.CheckPermission(ctx, "tag", &story); err != nil {
 		return nil, err
 	}
 
@@ -265,9 +291,9 @@ func (s *StoryService) EditChapter(ctx context.Context, chapter *models.Chapter,
 	}
 
 	if story, err := s.stories.Find(ctx, chapter.StoryID); err == nil {
-		s.changeService.Submit(ctx, "Comment", "add", story.Listed, changekeys.Many(story, chapter), chapter)
+		s.changeService.Submit(ctx, "Chapter", "edit", story.Listed, changekeys.Many(story, chapter), chapter)
 	} else {
-		s.changeService.Submit(ctx, "Comment", "add", false, changekeys.Many(models.Story{ID: chapter.StoryID}, chapter), chapter)
+		s.changeService.Submit(ctx, "Chapter", "edit", false, changekeys.Many(models.Story{ID: chapter.StoryID}, chapter), chapter)
 	}
 
 	return chapter, nil