Long overdue initial commit

model/log/log.go

@@ -0,0 +1,374 @@
+package log
+
+import (
+	"errors"
+	"fmt"
+	"log"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"git.aiterp.net/rpdata/api/internal/store"
+	"git.aiterp.net/rpdata/api/model/character"
+	"git.aiterp.net/rpdata/api/model/counter"
+
+	"github.com/globalsign/mgo/bson"
+
+	"github.com/globalsign/mgo"
+)
+
+var postMutex sync.RWMutex
+
+var characterUpdateMutex sync.Mutex
+
+var logsCollection *mgo.Collection
+
+// Log is the header/session for a log file.
+type Log struct {
+	ID           string    `bson:"_id"`
+	ShortID      string    `bson:"shortId"`
+	Date         time.Time `bson:"date"`
+	Channel      string    `bson:"channel"`
+	Title        string    `bson:"title,omitempty"`
+	Event        string    `bson:"event,omitempty"`
+	Description  string    `bson:"description,omitempty"`
+	Open         bool      `bson:"open"`
+	CharacterIDs []string  `bson:"characterIds"`
+}
+
+// New creates a new Log
+func New(date time.Time, channel, title, event, description string, open bool) (Log, error) {
+	nextID, err := counter.Next("auto_increment", "Log")
+	if err != nil {
+		return Log{}, err
+	}
+
+	log := Log{
+		ID:           MakeLogID(date, channel),
+		ShortID:      "L" + strconv.Itoa(nextID),
+		Date:         date,
+		Channel:      channel,
+		Title:        title,
+		Event:        event,
+		Description:  description,
+		Open:         open,
+		CharacterIDs: nil,
+	}
+
+	err = logsCollection.Insert(log)
+	if err != nil {
+		return Log{}, err
+	}
+
+	return log, nil
+}
+
+// FindID finds a log either by it's ID or short ID.
+func FindID(id string) (Log, error) {
+	return findLog(bson.M{
+		"$or": []bson.M{
+			bson.M{"_id": id},
+			bson.M{"shortId": id},
+		},
+	})
+}
+
+// List lists all logs
+func List(limit int) ([]Log, error) {
+	return listLog(bson.M{}, limit)
+}
+
+// Remove removes the log post with this ID. Both the long and short ID is accepted
+func Remove(id string) error {
+	return logsCollection.Remove(bson.M{
+		"$or": []bson.M{
+			bson.M{"_id": id},
+			bson.M{"shortId": id},
+		},
+	})
+}
+
+// ListSearch lists the logs matching the parameters. Empty/zero values means the parameter is ingored when
+// building the query. This is the old aitelogs2 way, but with the addition of a text search.
+//
+// If a text search is specified, it will make two trips to the database.
+func ListSearch(textSearch string, channels []string, characterIds []string, events []string, open bool, limit int) ([]Log, error) {
+	postMutex.RLock()
+	defer postMutex.RUnlock()
+
+	query := bson.M{}
+
+	// Run a text search
+	if textSearch != "" {
+		searchResults := make([]string, 0, 32)
+
+		err := postCollection.Find(bson.M{"$text": bson.M{"$search": textSearch}}).Distinct("logId", &searchResults)
+		if err != nil {
+			return nil, err
+		}
+
+		// Posts always use shortId to refer to the log
+		query["shortId"] = bson.M{"$in": searchResults}
+	}
+
+	// Find logs including any of the specified events and channels
+	if len(channels) > 0 {
+		query["channel"] = bson.M{"$in": channels}
+	}
+	if len(events) > 0 {
+		query["events"] = bson.M{"$in": channels}
+	}
+
+	// Find logs including all of the specified character IDs.
+	if len(characterIds) > 0 {
+		query["characterIds"] = bson.M{"$all": characterIds}
+	}
+
+	// Limit to only open logs
+	if open {
+		query["open"] = true
+	}
+
+	return listLog(query, limit)
+}
+
+// Edit sets the metadata
+func (log *Log) Edit(title *string, event *string, description *string, open *bool) error {
+	changes := bson.M{}
+
+	if title != nil && *title != log.Title {
+		changes["title"] = *title
+	}
+	if event != nil && *event != log.Event {
+		changes["event"] = *event
+	}
+	if description != nil && *description != log.Description {
+		changes["description"] = *description
+	}
+	if open != nil && *open != log.Open {
+		changes["open"] = *open
+	}
+
+	if len(changes) == 0 {
+		return nil
+	}
+
+	err := logsCollection.UpdateId(log.ID, bson.M{"$set": changes})
+	if err != nil {
+		return err
+	}
+
+	if title != nil {
+		log.Title = *title
+	}
+	if event != nil {
+		log.Event = *event
+	}
+	if description != nil {
+		log.Description = *description
+	}
+	if open != nil {
+		log.Open = *open
+	}
+
+	return nil
+}
+
+// Posts gets all the posts under the log. If no kinds are specified, it
+// will get all posts
+func (log *Log) Posts(kinds ...string) ([]Post, error) {
+	postMutex.RLock()
+	defer postMutex.RUnlock()
+
+	query := bson.M{
+		"$or": []bson.M{
+			bson.M{"logId": log.ID},
+			bson.M{"logId": log.ShortID},
+		},
+	}
+
+	if len(kinds) > 0 {
+		for i := range kinds {
+			kinds[i] = strings.ToLower(kinds[i])
+		}
+
+		query["kind"] = bson.M{"$in": kinds}
+	}
+
+	posts, err := listPosts(query)
+	if err != nil {
+		return nil, err
+	}
+
+	sort.SliceStable(posts, func(i, j int) bool {
+		return posts[i].Index < posts[j].Index
+	})
+
+	return posts, nil
+}
+
+// NewPost creates a new post.
+func (log *Log) NewPost(time time.Time, kind, nick, text string) (Post, error) {
+	if kind == "" || nick == "" || text == "" {
+		return Post{}, errors.New("Missing/empty parameters")
+	}
+
+	postMutex.RLock()
+	defer postMutex.RUnlock()
+
+	index, err := counter.Next("next_post_id", log.ShortID)
+	if err != nil {
+		return Post{}, err
+	}
+
+	post := Post{
+		ID:    MakePostID(time),
+		Index: index,
+		LogID: log.ShortID,
+		Time:  time,
+		Kind:  kind,
+		Nick:  nick,
+		Text:  text,
+	}
+
+	err = postCollection.Insert(post)
+	if err != nil {
+		return Post{}, err
+	}
+
+	return post, nil
+}
+
+// UpdateCharacters updates the character list
+func (log *Log) UpdateCharacters() error {
+	characterUpdateMutex.Lock()
+	defer characterUpdateMutex.Unlock()
+
+	posts, err := log.Posts()
+	if err != nil {
+		return err
+	}
+
+	added := make(map[string]bool)
+	removed := make(map[string]bool)
+	for _, post := range posts {
+		if post.Kind == "text" || post.Kind == "action" {
+			if strings.HasPrefix(post.Text, "(") || strings.Contains(post.Nick, "(") || strings.Contains(post.Nick, "[E]") {
+				continue
+			}
+
+			// Clean up the nick (remove possessive suffix, comma, formatting stuff)
+			if strings.HasSuffix(post.Nick, "'s") || strings.HasSuffix(post.Nick, "`s") {
+				post.Nick = post.Nick[:len(post.Nick)-2]
+			} else if strings.HasSuffix(post.Nick, "'") || strings.HasSuffix(post.Nick, "`") || strings.HasSuffix(post.Nick, ",") || strings.HasSuffix(post.Nick, "\x0f") {
+				post.Nick = post.Nick[:len(post.Nick)-1]
+			}
+
+			added[post.Nick] = true
+		}
+		if post.Kind == "chars" {
+			tokens := strings.Fields(post.Text)
+			for _, token := range tokens {
+				if strings.HasPrefix(token, "-") {
+					removed[token[1:]] = true
+				} else {
+					added[strings.Replace(token, "+", "", 1)] = true
+				}
+			}
+		}
+	}
+
+	nicks := make([]string, 0, len(added))
+	for nick := range added {
+		if added[nick] && !removed[nick] {
+			nicks = append(nicks, nick)
+		}
+	}
+
+	characters, err := character.ListNicks(nicks...)
+	if err != nil {
+		return err
+	}
+
+	characterIDs := make([]string, len(characters))
+	for i, char := range characters {
+		characterIDs[i] = char.ID
+	}
+
+	err = logsCollection.UpdateId(log.ID, bson.M{"$set": bson.M{"characterIds": characterIDs}})
+	if err != nil {
+		return err
+	}
+
+	for _, nick := range nicks {
+		found := false
+
+		for _, character := range characters {
+			if character.HasNick(nick) {
+				found = true
+				break
+			}
+		}
+
+		if !found {
+			addUnknownNick(nick)
+		}
+	}
+
+	log.CharacterIDs = characterIDs
+
+	return nil
+}
+
+func findLog(query interface{}) (Log, error) {
+	log := Log{}
+	err := logsCollection.Find(query).One(&log)
+	if err != nil {
+		return Log{}, err
+	}
+
+	return log, nil
+}
+
+func listLog(query interface{}, limit int) ([]Log, error) {
+	logs := make([]Log, 0, 64)
+	err := logsCollection.Find(query).Limit(limit).Sort("-date").All(&logs)
+	if err != nil {
+		return nil, err
+	}
+
+	return logs, nil
+}
+
+func iterLogs(query interface{}, limit int) *mgo.Iter {
+	return logsCollection.Find(query).Sort("-date").Limit(limit).Batch(8).Iter()
+}
+
+// MakeLogID generates log IDs that are of the format from logbot2, though it will break compatibility.
+func MakeLogID(date time.Time, channel string) string {
+	return fmt.Sprintf("%s%03d_%s", date.UTC().Format("2006-01-02_150405"), (date.Nanosecond() / int(time.Millisecond/time.Nanosecond)), channel[1:])
+}
+
+func init() {
+	store.HandleInit(func(db *mgo.Database) {
+		logsCollection = db.C("logbot3.logs")
+
+		logsCollection.EnsureIndexKey("date")
+		logsCollection.EnsureIndexKey("channel")
+		logsCollection.EnsureIndexKey("characterIds")
+		logsCollection.EnsureIndexKey("event")
+		logsCollection.EnsureIndex(mgo.Index{
+			Key: []string{"channel", "open"},
+		})
+		err := logsCollection.EnsureIndex(mgo.Index{
+			Key:      []string{"shortId"},
+			Unique:   true,
+			DropDups: true,
+		})
+		if err != nil {
+			log.Fatalln("init logbot3.logs:", err)
+		}
+	})
+}

model/log/log_test.go

@@ -0,0 +1,83 @@
+package log_test
+
+import (
+	"testing"
+	"time"
+
+	"git.aiterp.net/rpdata/api/internal/store"
+	"git.aiterp.net/rpdata/api/model/log"
+)
+
+func TestMakeLogID(t *testing.T) {
+	table := []struct {
+		Date     time.Time
+		Channel  string
+		Expected string
+	}{
+		{
+			time.Date(2018, 4, 9, 9, 3, 0, 133000000, time.FixedZone("CEST", 7200)), "#Miner'sRespite",
+			"2018-04-09_070300133_Miner'sRespite",
+		},
+		{
+			time.Date(2017, 3, 23, 23, 59, 59, 0, time.UTC), "#RedrockAgency",
+			"2017-03-23_235959000_RedrockAgency",
+		},
+	}
+
+	for _, row := range table {
+		t.Run(row.Expected, func(t *testing.T) {
+			id := log.MakeLogID(row.Date, row.Channel)
+
+			if id != row.Expected {
+				t.Error("Failed to make ID, result:", id)
+			}
+		})
+	}
+}
+
+func TestSearch(t *testing.T) {
+	store.Init()
+
+	logs, err := log.ListSearch("", nil, []string{"C31", "C51"}, nil, false, 0)
+	if err != nil {
+		t.Log(err)
+		t.Skip()
+	}
+
+	for _, l := range logs {
+		t.Log(l.ID)
+	}
+}
+
+func TestMovePost(t *testing.T) {
+	store.Init()
+
+	l, err := log.FindID("L684")
+	if err != nil {
+		t.Log(err)
+		t.Skip()
+	}
+
+	posts, err := l.Posts()
+	if err != nil {
+		t.Log(err)
+		t.Skip()
+	}
+
+	for _, post := range posts {
+		if post.ID == "blfn5uaxpyf11j4phxo" {
+			start := time.Now()
+
+			err := post.Move(1)
+			if err != nil {
+				t.Error(err)
+			}
+
+			t.Log(time.Since(start))
+		}
+	}
+}
+
+func TestMakePostID(t *testing.T) {
+	t.Log(log.MakePostID(time.Now()))
+}

model/log/post.go

@@ -0,0 +1,198 @@
+package log
+
+import (
+	"crypto/rand"
+	"encoding/binary"
+	"errors"
+	"log"
+	"strconv"
+	"time"
+
+	"git.aiterp.net/rpdata/api/internal/store"
+	"github.com/globalsign/mgo"
+	"github.com/globalsign/mgo/bson"
+)
+
+var postCollection *mgo.Collection
+
+// A Post is a part of a log file.
+type Post struct {
+	ID    string    `bson:"_id"`
+	LogID string    `bson:"logId"`
+	Time  time.Time `bson:"time"`
+	Kind  string    `bson:"kind"`
+	Nick  string    `bson:"nick"`
+	Text  string    `bson:"text"`
+	Index int       `bson:"index"`
+}
+
+// Edit the post
+func (post *Post) Edit(time *time.Time, kind *string, nick *string, text *string) error {
+	changes := bson.M{}
+	changed := false
+	postCopy := *post
+
+	if time != nil && !time.IsZero() && !time.Equal(post.Time) {
+		changes["time"] = *time
+		changed = true
+		postCopy.Time = *time
+	}
+	if kind != nil && *kind != "" && *kind != post.Kind {
+		changes["kind"] = *kind
+		changed = true
+		postCopy.Kind = *kind
+	}
+	if nick != nil && *nick != "" && *nick != post.Nick {
+		changes["nick"] = *nick
+		changed = true
+		postCopy.Nick = *nick
+	}
+	if text != nil && *text != "" && *text != post.Text {
+		changes["text"] = *text
+		changed = true
+		postCopy.Text = *text
+	}
+
+	if !changed {
+		return nil
+	}
+
+	err := postCollection.UpdateId(post.ID, bson.M{"$set": changes})
+	if err != nil {
+		return err
+	}
+
+	*post = postCopy
+	return nil
+}
+
+// Move the post
+func (post *Post) Move(targetIndex int) error {
+	if targetIndex < 1 {
+		return errors.New("Invalid index")
+	}
+
+	postMutex.Lock()
+	defer postMutex.Unlock()
+
+	// To avoid problems, only allow target indices that are allowed. If it's 1, then there is bound to
+	// be a post at the index.
+	if targetIndex > 1 {
+		existingPost := Post{}
+		err := postCollection.Find(bson.M{"logId": post.LogID, "index": targetIndex}).One(&existingPost)
+
+		if err != nil || existingPost.Index != targetIndex {
+			return errors.New("No post found at the index")
+		}
+	}
+
+	query := bson.M{"logId": post.LogID}
+	operation := bson.M{"$inc": bson.M{"index": 1}}
+
+	if targetIndex < post.Index {
+		query["$and"] = []bson.M{
+			bson.M{"index": bson.M{"$gte": targetIndex}},
+			bson.M{"index": bson.M{"$lt": post.Index}},
+		}
+	} else {
+		query["$and"] = []bson.M{
+			bson.M{"index": bson.M{"$gt": post.Index}},
+			bson.M{"index": bson.M{"$lte": targetIndex}},
+		}
+
+		operation["$inc"] = bson.M{"index": -1}
+	}
+
+	_, err := postCollection.UpdateAll(query, operation)
+	if err != nil {
+		return errors.New("moving others: " + err.Error())
+	}
+
+	err = postCollection.UpdateId(post.ID, bson.M{"$set": bson.M{"index": targetIndex}})
+	if err != nil {
+		return errors.New("moving: " + err.Error())
+	}
+
+	post.Index = targetIndex
+
+	return nil
+}
+
+// FindPostID finds a log post by ID.
+func FindPostID(id string) (Post, error) {
+	return findPost(bson.M{"_id": id})
+}
+
+// ListPostIDs lists log posts by ID
+func ListPostIDs(ids ...string) ([]Post, error) {
+	return listPosts(bson.M{"_id": bson.M{"$in": ids}})
+}
+
+// RemovePost removes a post, moving all subsequent post up one index
+func RemovePost(id string) (Post, error) {
+	postMutex.Lock()
+	defer postMutex.Unlock()
+
+	post, err := findPost(bson.M{"_id": id})
+	if err != nil {
+		return Post{}, err
+	}
+
+	err = postCollection.RemoveId(id)
+	if err != nil {
+		return Post{}, err
+	}
+
+	_, err = postCollection.UpdateAll(bson.M{"logId": post.LogID, "index": bson.M{"$gt": post.Index}}, bson.M{"$inc": bson.M{"index": -1}})
+	if err != nil {
+		return Post{}, err
+	}
+
+	return post, nil
+}
+
+func findPost(query interface{}) (Post, error) {
+	post := Post{}
+	err := postCollection.Find(query).One(&post)
+	if err != nil {
+		return Post{}, err
+	}
+
+	return post, nil
+}
+
+func listPosts(query interface{}) ([]Post, error) {
+	posts := make([]Post, 0, 64)
+	err := postCollection.Find(query).All(&posts)
+	if err != nil {
+		return nil, err
+	}
+
+	return posts, nil
+}
+
+// MakePostID makes a random post ID
+func MakePostID(time time.Time) string {
+	data := make([]byte, 4)
+	rand.Read(data)
+
+	return "P" + strconv.FormatInt(time.UnixNano(), 36) + strconv.FormatInt(int64(binary.LittleEndian.Uint32(data)), 36)
+}
+
+func init() {
+	store.HandleInit(func(db *mgo.Database) {
+		postCollection = db.C("logbot3.posts")
+
+		postCollection.EnsureIndexKey("logId")
+		postCollection.EnsureIndexKey("time")
+		postCollection.EnsureIndexKey("kind")
+		postCollection.EnsureIndexKey("index")
+
+		err := postCollection.EnsureIndex(mgo.Index{
+			Key: []string{"$text:text"},
+		})
+		if err != nil {
+			log.Fatalln("init logbot3.logs:", err)
+		}
+	})
+}

model/log/unknownnick.go

@@ -0,0 +1,42 @@
+package log
+
+import (
+	"git.aiterp.net/rpdata/api/internal/store"
+	"github.com/globalsign/mgo"
+	"github.com/globalsign/mgo/bson"
+)
+
+var unknownConnection *mgo.Collection
+
+// An UnknownNick is a nick found by the character list updater that
+// does not exist. The score is the number of logs that nick was in, meaning
+// nicks with a higher score should be a high priority to be matched with
+// a character.
+type UnknownNick struct {
+	Nick  string `bson:"_id" json:"nick"`
+	Score int    `bson:"score" json:"score"`
+}
+
+// UnknownNicks gets all the unknown nicks from the last search.
+func UnknownNicks() ([]UnknownNick, error) {
+	nicks := make([]UnknownNick, 0, 256)
+	err := unknownConnection.Find(bson.M{}).Sort("-score").All(&nicks)
+
+	return nicks, err
+}
+
+func addUnknownNick(nick string) error {
+	_, err := unknownConnection.UpsertId(nick, bson.M{"$inc": bson.M{"score": 1}})
+	return err
+}
+
+func clearUnknownNicks() error {
+	_, err := unknownConnection.RemoveAll(bson.M{})
+	return err
+}
+
+func init() {
+	store.HandleInit(func(db *mgo.Database) {
+		unknownConnection = db.C("logbot3.unknown_nicks")
+	})
+}

model/log/updater.go

@@ -0,0 +1,84 @@
+package log
+
+import (
+	"sync"
+	"time"
+
+	"github.com/globalsign/mgo/bson"
+)
+
+var scheduleCharacterUpdate = func() func() {
+	var mutex sync.Mutex
+	var scheduled bool
+
+	return func() {
+		mutex.Lock()
+		if !scheduled {
+			go func() {
+				time.Sleep(time.Second * 60)
+
+				// If another comes along in the next 2-3 seconds, it should schedule a new
+				// round to avoid a character only appearing in half their logs.
+				mutex.Lock()
+				scheduled = false
+				mutex.Unlock()
+
+				UpdateAllCharacters()
+			}()
+
+			scheduled = true
+		}
+		mutex.Unlock()
+	}
+}()
+
+// ScheduleCharacterUpdate schedules a full update within the minute.
+// Subsequent calls within that time will not schedule anything. Even
+// if the operation takes a few seconds at most, it need not be ran often.
+func ScheduleCharacterUpdate() {
+	scheduleCharacterUpdate()
+}
+
+// UpdateCharacters is a shorthand for getting a log and updaing its characters
+func UpdateCharacters(logID string) error {
+	log, err := FindID(logID)
+	if err != nil {
+		return err
+	}
+
+	return log.UpdateCharacters()
+}
+
+// UpdateAllCharacters updates character list on all logs. This should
+// be done if one or more characters failed to be added.
+func UpdateAllCharacters() (updated int, err error) {
+	updated = 0
+
+	err = clearUnknownNicks()
+	if err != nil {
+		return
+	}
+
+	iter := iterLogs(bson.M{}, 0)
+	err = iter.Err()
+	if err != nil {
+		return
+	}
+
+	log := Log{}
+	for iter.Next(&log) {
+		err = log.UpdateCharacters()
+		if err != nil {
+			return
+		}
+
+		updated++
+	}
+
+	err = iter.Err()
+	if err != nil {
+		return
+	}
+
+	return
+}