Long overdue initial commit

model/change/change.go

@@ -0,0 +1,75 @@
+package change
+
+import (
+	"log"
+	"time"
+
+	"git.aiterp.net/rpdata/api/model/counter"
+
+	"git.aiterp.net/rpdata/api/internal/store"
+	"github.com/globalsign/mgo"
+)
+
+var collection *mgo.Collection
+
+// A Change represents a change in any other model
+type Change struct {
+	ID       int         `bson:"_id" json:"id"`
+	Time     time.Time   `bson:"time" json:"time"`
+	Model    string      `bson:"model" json:"model"`
+	Op       string      `bson:"op" json:"op"`
+	Author   string      `bson:"author,omitempty" json:"author,omitempty"`
+	ObjectID string      `bson:"objectId,omitempty" json:"objectId,omitempty"`
+	Data     interface{} `bson:"data,omitempty" json:"data,omitempty"`
+}
+
+// PublicModels lists which models can be listed in bulk by anyone.
+var PublicModels = []string{
+	"Character",
+	"Log",
+	"Post",
+}
+
+// Submit submits a change to the history.
+func Submit(model, op, author, objectID string, data interface{}) (Change, error) {
+	index, err := counter.Next("auto_increment", "Change")
+	if err != nil {
+		return Change{}, err
+	}
+
+	change := Change{
+		ID:       index,
+		Time:     time.Now(),
+		Model:    model,
+		Op:       op,
+		Author:   author,
+		ObjectID: objectID,
+		Data:     data,
+	}
+
+	err = collection.Insert(&change)
+	if err != nil {
+		return Change{}, err
+	}
+
+	return change, err
+}
+
+func init() {
+	store.HandleInit(func(db *mgo.Database) {
+		collection = db.C("common.history")
+
+		collection.EnsureIndexKey("model")
+		collection.EnsureIndexKey("author")
+		collection.EnsureIndexKey("objectId")
+
+		err := collection.EnsureIndex(mgo.Index{
+			Name:        "expiry",
+			Key:         []string{"time"},
+			ExpireAfter: time.Hour * 336,
+		})
+		if err != nil {
+			log.Fatalln(err)
+		}
+	})
+}

model/character/character.go

@@ -0,0 +1,231 @@
+package character
+
+import (
+	"errors"
+	"log"
+	"strconv"
+	"strings"
+
+	"git.aiterp.net/rpdata/api/internal/store"
+	"git.aiterp.net/rpdata/api/model/counter"
+	"github.com/globalsign/mgo"
+	"github.com/globalsign/mgo/bson"
+)
+
+var collection *mgo.Collection
+
+// Character is a common data model representing an RP character or NPC.
+type Character struct {
+	ID          string   `json:"id" bson:"_id"`
+	Nicks       []string `json:"nicks" bson:"nicks"`
+	Name        string   `json:"name" bson:"name"`
+	ShortName   string   `json:"shortName" bson:"shortName"`
+	Author      string   `json:"author" bson:"author"`
+	Description string   `json:"description" bson:"description"`
+}
+
+// HasNick returns true if the character has that nick
+func (character *Character) HasNick(nick string) bool {
+	for i := range character.Nicks {
+		if strings.EqualFold(character.Nicks[i], nick) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// AddNick adds a nick to the character. It will return an error
+// if the nick already exists.
+func (character *Character) AddNick(nick string) error {
+	for i := range character.Nicks {
+		if strings.EqualFold(character.Nicks[i], nick) {
+			return errors.New("Nick already exists")
+		}
+	}
+
+	err := collection.UpdateId(character.ID, bson.M{"$push": bson.M{"nicks": nick}})
+	if err != nil {
+		return err
+	}
+
+	character.Nicks = append(character.Nicks, nick)
+
+	return nil
+}
+
+// RemoveNick removes the nick from the character. It will raise
+// an error if the nick does not exist; even if that kind of is
+// the end goal.
+func (character *Character) RemoveNick(nick string) error {
+	index := -1
+	for i := range character.Nicks {
+		if strings.EqualFold(character.Nicks[i], nick) {
+			index = i
+			break
+		}
+	}
+	if index == -1 {
+		return errors.New("Nick does not exist")
+	}
+
+	err := collection.UpdateId(character.ID, bson.M{"$pull": bson.M{"nicks": nick}})
+	if err != nil {
+		return err
+	}
+
+	character.Nicks = append(character.Nicks[:index], character.Nicks[index+1:]...)
+
+	return nil
+}
+
+// Edit sets the fields of metadata. Only non-empty and different fields will be set in the
+// database, preventing out of order edits to two fields from conflicting
+func (character *Character) Edit(name, shortName, description string) error {
+	changes := bson.M{}
+	if len(name) > 0 && name != character.Name {
+		changes["name"] = name
+	}
+	if len(shortName) > 0 && shortName != character.ShortName {
+		changes["shortName"] = shortName
+	}
+	if len(description) > 0 && description != character.Description {
+		changes["description"] = description
+	}
+
+	err := collection.UpdateId(character.ID, changes)
+	if err != nil {
+		return err
+	}
+
+	if changes["name"] != nil {
+		character.Name = name
+	}
+	if changes["shortName"] != nil {
+		character.ShortName = shortName
+	}
+	if changes["description"] != nil {
+		character.Description = description
+	}
+
+	return nil
+}
+
+// Remove removes the character from the database. The reason this is an instance method
+// is that it should only be done after an authorization check.
+func (character *Character) Remove() error {
+	return collection.RemoveId(character.ID)
+}
+
+// FindID finds Character by ID
+func FindID(id string) (Character, error) {
+	return find(bson.M{"_id": id})
+}
+
+// FindNick finds Character by nick
+func FindNick(nick string) (Character, error) {
+	return find(bson.M{"nicks": nick})
+}
+
+// FindName finds Character by either full name or
+// short name.
+func FindName(name string) (Character, error) {
+	return find(bson.M{"$or": []bson.M{bson.M{"name": name}, bson.M{"shortName": name}}})
+}
+
+// List lists all characters
+func List() ([]Character, error) {
+	return list(bson.M{})
+}
+
+// ListAuthor lists all characters by author
+func ListAuthor(author string) ([]Character, error) {
+	return list(bson.M{"author": author})
+}
+
+// ListNicks lists all characters with either of these nicks. This was made with
+// the logbot in mind, to batch an order for characters.
+func ListNicks(nicks ...string) ([]Character, error) {
+	return list(bson.M{"nicks": bson.M{"$in": nicks}})
+}
+
+// ListIDs lists all characters with either of these IDs.
+func ListIDs(ids ...string) ([]Character, error) {
+	return list(bson.M{"_id": bson.M{"$in": ids}})
+}
+
+// New creates a Character and pushes it to the database. It does some validation
+// on nick, name, shortName and author. Leave the shortname blank to have it be the
+// first name.
+func New(nick, name, shortName, author, description string) (Character, error) {
+	if len(nick) < 1 || len(name) < 1 || len(author) < 1 {
+		return Character{}, errors.New("Nick, name, or author name too short or empty")
+	}
+	if shortName == "" {
+		shortName = strings.SplitN(name, " ", 2)[0]
+	}
+
+	char, err := FindNick(nick)
+	if err == nil && char.ID != "" {
+		return Character{}, errors.New("Nick is occupied")
+	}
+
+	nextID, err := counter.Next("auto_increment", "Character")
+	if err != nil {
+		return Character{}, err
+	}
+
+	character := Character{
+		ID:          "C" + strconv.Itoa(nextID),
+		Nicks:       []string{nick},
+		Name:        name,
+		ShortName:   shortName,
+		Author:      author,
+		Description: description,
+	}
+
+	err = collection.Insert(character)
+	if err != nil {
+		return Character{}, err
+	}
+
+	return character, nil
+}
+
+func find(query interface{}) (Character, error) {
+	character := Character{}
+	err := collection.Find(query).One(&character)
+	if err != nil {
+		return Character{}, err
+	}
+
+	return character, nil
+}
+
+func list(query interface{}) ([]Character, error) {
+	characters := make([]Character, 0, 64)
+	err := collection.Find(query).All(&characters)
+	if err != nil {
+		return nil, err
+	}
+
+	return characters, nil
+}
+
+func init() {
+	store.HandleInit(func(db *mgo.Database) {
+		collection = db.C("common.characters")
+
+		collection.EnsureIndexKey("name")
+		collection.EnsureIndexKey("shortName")
+		collection.EnsureIndexKey("author")
+		err := collection.EnsureIndex(mgo.Index{
+			Key:      []string{"nicks"},
+			Unique:   true,
+			DropDups: true,
+		})
+		if err != nil {
+			log.Fatalln("init common.characters:", err)
+		}
+	})
+}

model/counter/counter.go

@@ -0,0 +1,37 @@
+package counter
+
+import (
+	"git.aiterp.net/rpdata/api/internal/store"
+	"github.com/globalsign/mgo"
+	"github.com/globalsign/mgo/bson"
+)
+
+var collection *mgo.Collection
+
+type counter struct {
+	ID    string `bson:"_id"`
+	Value int    `bson:"value"`
+}
+
+// Next gets the next value of a counter, or an error if it hasn't.
+func Next(category, name string) (int, error) {
+	id := category + "." + name
+	doc := counter{}
+
+	_, err := collection.Find(bson.M{"_id": id}).Apply(mgo.Change{
+		Update:    bson.M{"$inc": bson.M{"value": 1}},
+		Upsert:    true,
+		ReturnNew: true,
+	}, &doc)
+	if err != nil {
+		return -1, err
+	}
+
+	return doc.Value, nil
+}
+
+func init() {
+	store.HandleInit(func(db *mgo.Database) {
+		collection = db.C("core.counters")
+	})
+}

model/counter/counter_test.go

@@ -0,0 +1,19 @@
+package counter_test
+
+import (
+	"testing"
+
+	"git.aiterp.net/rpdata/api/internal/store"
+	"git.aiterp.net/rpdata/api/model/counter"
+)
+
+func TestCounter(t *testing.T) {
+	store.Init()
+
+	value, err := counter.Next("test", "times_tested")
+	if err != nil {
+		t.Error(err)
+	}
+
+	t.Log("Value:", value)
+}

model/log/log.go

@@ -0,0 +1,374 @@
+package log
+
+import (
+	"errors"
+	"fmt"
+	"log"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"git.aiterp.net/rpdata/api/internal/store"
+	"git.aiterp.net/rpdata/api/model/character"
+	"git.aiterp.net/rpdata/api/model/counter"
+
+	"github.com/globalsign/mgo/bson"
+
+	"github.com/globalsign/mgo"
+)
+
+var postMutex sync.RWMutex
+
+var characterUpdateMutex sync.Mutex
+
+var logsCollection *mgo.Collection
+
+// Log is the header/session for a log file.
+type Log struct {
+	ID           string    `bson:"_id"`
+	ShortID      string    `bson:"shortId"`
+	Date         time.Time `bson:"date"`
+	Channel      string    `bson:"channel"`
+	Title        string    `bson:"title,omitempty"`
+	Event        string    `bson:"event,omitempty"`
+	Description  string    `bson:"description,omitempty"`
+	Open         bool      `bson:"open"`
+	CharacterIDs []string  `bson:"characterIds"`
+}
+
+// New creates a new Log
+func New(date time.Time, channel, title, event, description string, open bool) (Log, error) {
+	nextID, err := counter.Next("auto_increment", "Log")
+	if err != nil {
+		return Log{}, err
+	}
+
+	log := Log{
+		ID:           MakeLogID(date, channel),
+		ShortID:      "L" + strconv.Itoa(nextID),
+		Date:         date,
+		Channel:      channel,
+		Title:        title,
+		Event:        event,
+		Description:  description,
+		Open:         open,
+		CharacterIDs: nil,
+	}
+
+	err = logsCollection.Insert(log)
+	if err != nil {
+		return Log{}, err
+	}
+
+	return log, nil
+}
+
+// FindID finds a log either by it's ID or short ID.
+func FindID(id string) (Log, error) {
+	return findLog(bson.M{
+		"$or": []bson.M{
+			bson.M{"_id": id},
+			bson.M{"shortId": id},
+		},
+	})
+}
+
+// List lists all logs
+func List(limit int) ([]Log, error) {
+	return listLog(bson.M{}, limit)
+}
+
+// Remove removes the log post with this ID. Both the long and short ID is accepted
+func Remove(id string) error {
+	return logsCollection.Remove(bson.M{
+		"$or": []bson.M{
+			bson.M{"_id": id},
+			bson.M{"shortId": id},
+		},
+	})
+}
+
+// ListSearch lists the logs matching the parameters. Empty/zero values means the parameter is ingored when
+// building the query. This is the old aitelogs2 way, but with the addition of a text search.
+//
+// If a text search is specified, it will make two trips to the database.
+func ListSearch(textSearch string, channels []string, characterIds []string, events []string, open bool, limit int) ([]Log, error) {
+	postMutex.RLock()
+	defer postMutex.RUnlock()
+
+	query := bson.M{}
+
+	// Run a text search
+	if textSearch != "" {
+		searchResults := make([]string, 0, 32)
+
+		err := postCollection.Find(bson.M{"$text": bson.M{"$search": textSearch}}).Distinct("logId", &searchResults)
+		if err != nil {
+			return nil, err
+		}
+
+		// Posts always use shortId to refer to the log
+		query["shortId"] = bson.M{"$in": searchResults}
+	}
+
+	// Find logs including any of the specified events and channels
+	if len(channels) > 0 {
+		query["channel"] = bson.M{"$in": channels}
+	}
+	if len(events) > 0 {
+		query["events"] = bson.M{"$in": channels}
+	}
+
+	// Find logs including all of the specified character IDs.
+	if len(characterIds) > 0 {
+		query["characterIds"] = bson.M{"$all": characterIds}
+	}
+
+	// Limit to only open logs
+	if open {
+		query["open"] = true
+	}
+
+	return listLog(query, limit)
+}
+
+// Edit sets the metadata
+func (log *Log) Edit(title *string, event *string, description *string, open *bool) error {
+	changes := bson.M{}
+
+	if title != nil && *title != log.Title {
+		changes["title"] = *title
+	}
+	if event != nil && *event != log.Event {
+		changes["event"] = *event
+	}
+	if description != nil && *description != log.Description {
+		changes["description"] = *description
+	}
+	if open != nil && *open != log.Open {
+		changes["open"] = *open
+	}
+
+	if len(changes) == 0 {
+		return nil
+	}
+
+	err := logsCollection.UpdateId(log.ID, bson.M{"$set": changes})
+	if err != nil {
+		return err
+	}
+
+	if title != nil {
+		log.Title = *title
+	}
+	if event != nil {
+		log.Event = *event
+	}
+	if description != nil {
+		log.Description = *description
+	}
+	if open != nil {
+		log.Open = *open
+	}
+
+	return nil
+}
+
+// Posts gets all the posts under the log. If no kinds are specified, it
+// will get all posts
+func (log *Log) Posts(kinds ...string) ([]Post, error) {
+	postMutex.RLock()
+	defer postMutex.RUnlock()
+
+	query := bson.M{
+		"$or": []bson.M{
+			bson.M{"logId": log.ID},
+			bson.M{"logId": log.ShortID},
+		},
+	}
+
+	if len(kinds) > 0 {
+		for i := range kinds {
+			kinds[i] = strings.ToLower(kinds[i])
+		}
+
+		query["kind"] = bson.M{"$in": kinds}
+	}
+
+	posts, err := listPosts(query)
+	if err != nil {
+		return nil, err
+	}
+
+	sort.SliceStable(posts, func(i, j int) bool {
+		return posts[i].Index < posts[j].Index
+	})
+
+	return posts, nil
+}
+
+// NewPost creates a new post.
+func (log *Log) NewPost(time time.Time, kind, nick, text string) (Post, error) {
+	if kind == "" || nick == "" || text == "" {
+		return Post{}, errors.New("Missing/empty parameters")
+	}
+
+	postMutex.RLock()
+	defer postMutex.RUnlock()
+
+	index, err := counter.Next("next_post_id", log.ShortID)
+	if err != nil {
+		return Post{}, err
+	}
+
+	post := Post{
+		ID:    MakePostID(time),
+		Index: index,
+		LogID: log.ShortID,
+		Time:  time,
+		Kind:  kind,
+		Nick:  nick,
+		Text:  text,
+	}
+
+	err = postCollection.Insert(post)
+	if err != nil {
+		return Post{}, err
+	}
+
+	return post, nil
+}
+
+// UpdateCharacters updates the character list
+func (log *Log) UpdateCharacters() error {
+	characterUpdateMutex.Lock()
+	defer characterUpdateMutex.Unlock()
+
+	posts, err := log.Posts()
+	if err != nil {
+		return err
+	}
+
+	added := make(map[string]bool)
+	removed := make(map[string]bool)
+	for _, post := range posts {
+		if post.Kind == "text" || post.Kind == "action" {
+			if strings.HasPrefix(post.Text, "(") || strings.Contains(post.Nick, "(") || strings.Contains(post.Nick, "[E]") {
+				continue
+			}
+
+			// Clean up the nick (remove possessive suffix, comma, formatting stuff)
+			if strings.HasSuffix(post.Nick, "'s") || strings.HasSuffix(post.Nick, "`s") {
+				post.Nick = post.Nick[:len(post.Nick)-2]
+			} else if strings.HasSuffix(post.Nick, "'") || strings.HasSuffix(post.Nick, "`") || strings.HasSuffix(post.Nick, ",") || strings.HasSuffix(post.Nick, "\x0f") {
+				post.Nick = post.Nick[:len(post.Nick)-1]
+			}
+
+			added[post.Nick] = true
+		}
+		if post.Kind == "chars" {
+			tokens := strings.Fields(post.Text)
+			for _, token := range tokens {
+				if strings.HasPrefix(token, "-") {
+					removed[token[1:]] = true
+				} else {
+					added[strings.Replace(token, "+", "", 1)] = true
+				}
+			}
+		}
+	}
+
+	nicks := make([]string, 0, len(added))
+	for nick := range added {
+		if added[nick] && !removed[nick] {
+			nicks = append(nicks, nick)
+		}
+	}
+
+	characters, err := character.ListNicks(nicks...)
+	if err != nil {
+		return err
+	}
+
+	characterIDs := make([]string, len(characters))
+	for i, char := range characters {
+		characterIDs[i] = char.ID
+	}
+
+	err = logsCollection.UpdateId(log.ID, bson.M{"$set": bson.M{"characterIds": characterIDs}})
+	if err != nil {
+		return err
+	}
+
+	for _, nick := range nicks {
+		found := false
+
+		for _, character := range characters {
+			if character.HasNick(nick) {
+				found = true
+				break
+			}
+		}
+
+		if !found {
+			addUnknownNick(nick)
+		}
+	}
+
+	log.CharacterIDs = characterIDs
+
+	return nil
+}
+
+func findLog(query interface{}) (Log, error) {
+	log := Log{}
+	err := logsCollection.Find(query).One(&log)
+	if err != nil {
+		return Log{}, err
+	}
+
+	return log, nil
+}
+
+func listLog(query interface{}, limit int) ([]Log, error) {
+	logs := make([]Log, 0, 64)
+	err := logsCollection.Find(query).Limit(limit).Sort("-date").All(&logs)
+	if err != nil {
+		return nil, err
+	}
+
+	return logs, nil
+}
+
+func iterLogs(query interface{}, limit int) *mgo.Iter {
+	return logsCollection.Find(query).Sort("-date").Limit(limit).Batch(8).Iter()
+}
+
+// MakeLogID generates log IDs that are of the format from logbot2, though it will break compatibility.
+func MakeLogID(date time.Time, channel string) string {
+	return fmt.Sprintf("%s%03d_%s", date.UTC().Format("2006-01-02_150405"), (date.Nanosecond() / int(time.Millisecond/time.Nanosecond)), channel[1:])
+}
+
+func init() {
+	store.HandleInit(func(db *mgo.Database) {
+		logsCollection = db.C("logbot3.logs")
+
+		logsCollection.EnsureIndexKey("date")
+		logsCollection.EnsureIndexKey("channel")
+		logsCollection.EnsureIndexKey("characterIds")
+		logsCollection.EnsureIndexKey("event")
+		logsCollection.EnsureIndex(mgo.Index{
+			Key: []string{"channel", "open"},
+		})
+		err := logsCollection.EnsureIndex(mgo.Index{
+			Key:      []string{"shortId"},
+			Unique:   true,
+			DropDups: true,
+		})
+		if err != nil {
+			log.Fatalln("init logbot3.logs:", err)
+		}
+	})
+}

model/log/log_test.go

@@ -0,0 +1,83 @@
+package log_test
+
+import (
+	"testing"
+	"time"
+
+	"git.aiterp.net/rpdata/api/internal/store"
+	"git.aiterp.net/rpdata/api/model/log"
+)
+
+func TestMakeLogID(t *testing.T) {
+	table := []struct {
+		Date     time.Time
+		Channel  string
+		Expected string
+	}{
+		{
+			time.Date(2018, 4, 9, 9, 3, 0, 133000000, time.FixedZone("CEST", 7200)), "#Miner'sRespite",
+			"2018-04-09_070300133_Miner'sRespite",
+		},
+		{
+			time.Date(2017, 3, 23, 23, 59, 59, 0, time.UTC), "#RedrockAgency",
+			"2017-03-23_235959000_RedrockAgency",
+		},
+	}
+
+	for _, row := range table {
+		t.Run(row.Expected, func(t *testing.T) {
+			id := log.MakeLogID(row.Date, row.Channel)
+
+			if id != row.Expected {
+				t.Error("Failed to make ID, result:", id)
+			}
+		})
+	}
+}
+
+func TestSearch(t *testing.T) {
+	store.Init()
+
+	logs, err := log.ListSearch("", nil, []string{"C31", "C51"}, nil, false, 0)
+	if err != nil {
+		t.Log(err)
+		t.Skip()
+	}
+
+	for _, l := range logs {
+		t.Log(l.ID)
+	}
+}
+
+func TestMovePost(t *testing.T) {
+	store.Init()
+
+	l, err := log.FindID("L684")
+	if err != nil {
+		t.Log(err)
+		t.Skip()
+	}
+
+	posts, err := l.Posts()
+	if err != nil {
+		t.Log(err)
+		t.Skip()
+	}
+
+	for _, post := range posts {
+		if post.ID == "blfn5uaxpyf11j4phxo" {
+			start := time.Now()
+
+			err := post.Move(1)
+			if err != nil {
+				t.Error(err)
+			}
+
+			t.Log(time.Since(start))
+		}
+	}
+}
+
+func TestMakePostID(t *testing.T) {
+	t.Log(log.MakePostID(time.Now()))
+}

model/log/post.go

@@ -0,0 +1,198 @@
+package log
+
+import (
+	"crypto/rand"
+	"encoding/binary"
+	"errors"
+	"log"
+	"strconv"
+	"time"
+
+	"git.aiterp.net/rpdata/api/internal/store"
+	"github.com/globalsign/mgo"
+	"github.com/globalsign/mgo/bson"
+)
+
+var postCollection *mgo.Collection
+
+// A Post is a part of a log file.
+type Post struct {
+	ID    string    `bson:"_id"`
+	LogID string    `bson:"logId"`
+	Time  time.Time `bson:"time"`
+	Kind  string    `bson:"kind"`
+	Nick  string    `bson:"nick"`
+	Text  string    `bson:"text"`
+	Index int       `bson:"index"`
+}
+
+// Edit the post
+func (post *Post) Edit(time *time.Time, kind *string, nick *string, text *string) error {
+	changes := bson.M{}
+	changed := false
+	postCopy := *post
+
+	if time != nil && !time.IsZero() && !time.Equal(post.Time) {
+		changes["time"] = *time
+		changed = true
+		postCopy.Time = *time
+	}
+	if kind != nil && *kind != "" && *kind != post.Kind {
+		changes["kind"] = *kind
+		changed = true
+		postCopy.Kind = *kind
+	}
+	if nick != nil && *nick != "" && *nick != post.Nick {
+		changes["nick"] = *nick
+		changed = true
+		postCopy.Nick = *nick
+	}
+	if text != nil && *text != "" && *text != post.Text {
+		changes["text"] = *text
+		changed = true
+		postCopy.Text = *text
+	}
+
+	if !changed {
+		return nil
+	}
+
+	err := postCollection.UpdateId(post.ID, bson.M{"$set": changes})
+	if err != nil {
+		return err
+	}
+
+	*post = postCopy
+	return nil
+}
+
+// Move the post
+func (post *Post) Move(targetIndex int) error {
+	if targetIndex < 1 {
+		return errors.New("Invalid index")
+	}
+
+	postMutex.Lock()
+	defer postMutex.Unlock()
+
+	// To avoid problems, only allow target indices that are allowed. If it's 1, then there is bound to
+	// be a post at the index.
+	if targetIndex > 1 {
+		existingPost := Post{}
+		err := postCollection.Find(bson.M{"logId": post.LogID, "index": targetIndex}).One(&existingPost)
+
+		if err != nil || existingPost.Index != targetIndex {
+			return errors.New("No post found at the index")
+		}
+	}
+
+	query := bson.M{"logId": post.LogID}
+	operation := bson.M{"$inc": bson.M{"index": 1}}
+
+	if targetIndex < post.Index {
+		query["$and"] = []bson.M{
+			bson.M{"index": bson.M{"$gte": targetIndex}},
+			bson.M{"index": bson.M{"$lt": post.Index}},
+		}
+	} else {
+		query["$and"] = []bson.M{
+			bson.M{"index": bson.M{"$gt": post.Index}},
+			bson.M{"index": bson.M{"$lte": targetIndex}},
+		}
+
+		operation["$inc"] = bson.M{"index": -1}
+	}
+
+	_, err := postCollection.UpdateAll(query, operation)
+	if err != nil {
+		return errors.New("moving others: " + err.Error())
+	}
+
+	err = postCollection.UpdateId(post.ID, bson.M{"$set": bson.M{"index": targetIndex}})
+	if err != nil {
+		return errors.New("moving: " + err.Error())
+	}
+
+	post.Index = targetIndex
+
+	return nil
+}
+
+// FindPostID finds a log post by ID.
+func FindPostID(id string) (Post, error) {
+	return findPost(bson.M{"_id": id})
+}
+
+// ListPostIDs lists log posts by ID
+func ListPostIDs(ids ...string) ([]Post, error) {
+	return listPosts(bson.M{"_id": bson.M{"$in": ids}})
+}
+
+// RemovePost removes a post, moving all subsequent post up one index
+func RemovePost(id string) (Post, error) {
+	postMutex.Lock()
+	defer postMutex.Unlock()
+
+	post, err := findPost(bson.M{"_id": id})
+	if err != nil {
+		return Post{}, err
+	}
+
+	err = postCollection.RemoveId(id)
+	if err != nil {
+		return Post{}, err
+	}
+
+	_, err = postCollection.UpdateAll(bson.M{"logId": post.LogID, "index": bson.M{"$gt": post.Index}}, bson.M{"$inc": bson.M{"index": -1}})
+	if err != nil {
+		return Post{}, err
+	}
+
+	return post, nil
+}
+
+func findPost(query interface{}) (Post, error) {
+	post := Post{}
+	err := postCollection.Find(query).One(&post)
+	if err != nil {
+		return Post{}, err
+	}
+
+	return post, nil
+}
+
+func listPosts(query interface{}) ([]Post, error) {
+	posts := make([]Post, 0, 64)
+	err := postCollection.Find(query).All(&posts)
+	if err != nil {
+		return nil, err
+	}
+
+	return posts, nil
+}
+
+// MakePostID makes a random post ID
+func MakePostID(time time.Time) string {
+	data := make([]byte, 4)
+	rand.Read(data)
+
+	return "P" + strconv.FormatInt(time.UnixNano(), 36) + strconv.FormatInt(int64(binary.LittleEndian.Uint32(data)), 36)
+}
+
+func init() {
+	store.HandleInit(func(db *mgo.Database) {
+		postCollection = db.C("logbot3.posts")
+
+		postCollection.EnsureIndexKey("logId")
+		postCollection.EnsureIndexKey("time")
+		postCollection.EnsureIndexKey("kind")
+		postCollection.EnsureIndexKey("index")
+
+		err := postCollection.EnsureIndex(mgo.Index{
+			Key: []string{"$text:text"},
+		})
+		if err != nil {
+			log.Fatalln("init logbot3.logs:", err)
+		}
+	})
+}

model/log/unknownnick.go

@@ -0,0 +1,42 @@
+package log
+
+import (
+	"git.aiterp.net/rpdata/api/internal/store"
+	"github.com/globalsign/mgo"
+	"github.com/globalsign/mgo/bson"
+)
+
+var unknownConnection *mgo.Collection
+
+// An UnknownNick is a nick found by the character list updater that
+// does not exist. The score is the number of logs that nick was in, meaning
+// nicks with a higher score should be a high priority to be matched with
+// a character.
+type UnknownNick struct {
+	Nick  string `bson:"_id" json:"nick"`
+	Score int    `bson:"score" json:"score"`
+}
+
+// UnknownNicks gets all the unknown nicks from the last search.
+func UnknownNicks() ([]UnknownNick, error) {
+	nicks := make([]UnknownNick, 0, 256)
+	err := unknownConnection.Find(bson.M{}).Sort("-score").All(&nicks)
+
+	return nicks, err
+}
+
+func addUnknownNick(nick string) error {
+	_, err := unknownConnection.UpsertId(nick, bson.M{"$inc": bson.M{"score": 1}})
+	return err
+}
+
+func clearUnknownNicks() error {
+	_, err := unknownConnection.RemoveAll(bson.M{})
+	return err
+}
+
+func init() {
+	store.HandleInit(func(db *mgo.Database) {
+		unknownConnection = db.C("logbot3.unknown_nicks")
+	})
+}

model/log/updater.go

@@ -0,0 +1,84 @@
+package log
+
+import (
+	"sync"
+	"time"
+
+	"github.com/globalsign/mgo/bson"
+)
+
+var scheduleCharacterUpdate = func() func() {
+	var mutex sync.Mutex
+	var scheduled bool
+
+	return func() {
+		mutex.Lock()
+		if !scheduled {
+			go func() {
+				time.Sleep(time.Second * 60)
+
+				// If another comes along in the next 2-3 seconds, it should schedule a new
+				// round to avoid a character only appearing in half their logs.
+				mutex.Lock()
+				scheduled = false
+				mutex.Unlock()
+
+				UpdateAllCharacters()
+			}()
+
+			scheduled = true
+		}
+		mutex.Unlock()
+	}
+}()
+
+// ScheduleCharacterUpdate schedules a full update within the minute.
+// Subsequent calls within that time will not schedule anything. Even
+// if the operation takes a few seconds at most, it need not be ran often.
+func ScheduleCharacterUpdate() {
+	scheduleCharacterUpdate()
+}
+
+// UpdateCharacters is a shorthand for getting a log and updaing its characters
+func UpdateCharacters(logID string) error {
+	log, err := FindID(logID)
+	if err != nil {
+		return err
+	}
+
+	return log.UpdateCharacters()
+}
+
+// UpdateAllCharacters updates character list on all logs. This should
+// be done if one or more characters failed to be added.
+func UpdateAllCharacters() (updated int, err error) {
+	updated = 0
+
+	err = clearUnknownNicks()
+	if err != nil {
+		return
+	}
+
+	iter := iterLogs(bson.M{}, 0)
+	err = iter.Err()
+	if err != nil {
+		return
+	}
+
+	log := Log{}
+	for iter.Next(&log) {
+		err = log.UpdateCharacters()
+		if err != nil {
+			return
+		}
+
+		updated++
+	}
+
+	err = iter.Err()
+	if err != nil {
+		return
+	}
+
+	return
+}