Browse Source

Fixed log service bugs with importing and character list not updating.

thegreatrefactor
Gisle Aune 5 years ago
parent
commit
51b2c9bfe0
  1. 2
      database/mongodb/posts.go
  2. 55
      models/logs/add.go
  3. 48
      models/logs/db.go
  4. 50
      models/logs/edit.go
  5. 16
      models/logs/find.go
  6. 70
      models/logs/list.go
  7. 22
      models/logs/remove.go
  8. 138
      models/logs/update-characters.go
  9. 5
      services/channels.go
  10. 43
      services/logs.go

2
database/mongodb/posts.go

@ -130,7 +130,7 @@ func (r *postRepository) InsertMany(ctx context.Context, posts ...*models.Post)
} }
logId := posts[0].LogID logId := posts[0].LogID
for _, post := range posts[1:] {
for _, post := range posts {
if post.LogID != logId { if post.LogID != logId {
return nil, repositories.ErrParentMismatch return nil, repositories.ErrParentMismatch
} }

55
models/logs/add.go

@ -1,55 +0,0 @@
package logs
import (
"errors"
"strconv"
"time"
"git.aiterp.net/rpdata/api/internal/counter"
"git.aiterp.net/rpdata/api/models"
"git.aiterp.net/rpdata/api/models/channels"
"github.com/globalsign/mgo/bson"
)
// Add creates a new Log
func Add(date time.Time, channelName, title, eventName, description string, open bool) (models.Log, error) {
nextID, err := counter.Next("auto_increment", "Log")
if err != nil {
return models.Log{}, errors.New("Failed to allocate short ID: " + err.Error())
}
_, err = channels.Ensure(channelName, open)
if err != nil {
return models.Log{}, errors.New("Failed to ensure channel: " + err.Error())
}
log := models.Log{
ID: makeLogID(date, channelName),
ShortID: "L" + strconv.Itoa(nextID),
Date: date,
ChannelName: channelName,
Title: title,
EventName: eventName,
Description: description,
Open: open,
CharacterIDs: nil,
}
err = collection.Insert(log)
if err != nil {
return models.Log{}, err
}
// There can be only one open log in the same channel. TODO: Transaction
if open {
query := bson.M{
"_id": bson.M{"$ne": log.ID},
"open": true,
"channel": log.ChannelName,
}
go collection.UpdateAll(query, bson.M{"$set": bson.M{"open": false}})
}
return log, nil
}

48
models/logs/db.go

@ -1,48 +0,0 @@
package logs
import (
"fmt"
"time"
"git.aiterp.net/rpdata/api/internal/store"
"git.aiterp.net/rpdata/api/models"
"github.com/globalsign/mgo"
)
var collection *mgo.Collection
var postCollection *mgo.Collection
func find(query interface{}) (models.Log, error) {
log := models.Log{}
err := collection.Find(query).One(&log)
if err != nil {
return models.Log{}, err
}
return log, nil
}
func list(query interface{}, limit int) ([]models.Log, error) {
logs := make([]models.Log, 0, 64)
err := collection.Find(query).Limit(limit).Sort("-date").All(&logs)
if err != nil {
return nil, err
}
return logs, nil
}
func iter(query interface{}, limit int) *mgo.Iter {
return collection.Find(query).Sort("-date").Limit(limit).Batch(8).Iter()
}
func makeLogID(date time.Time, channel string) string {
return fmt.Sprintf("%s%03d_%s", date.UTC().Format("2006-01-02_150405"), (date.Nanosecond() / int(time.Millisecond/time.Nanosecond)), channel[1:])
}
func init() {
store.HandleInit(func(db *mgo.Database) {
collection = db.C("logbot3.logs")
postCollection = db.C("logbot3.posts")
})
}

50
models/logs/edit.go

@ -1,50 +0,0 @@
package logs
import (
"git.aiterp.net/rpdata/api/models"
"github.com/globalsign/mgo/bson"
)
// Edit sets a log's meta-data.
func Edit(log models.Log, title *string, event *string, description *string, open *bool) (models.Log, error) {
changes := bson.M{}
if title != nil && *title != log.Title {
changes["title"] = *title
log.Title = *title
}
if event != nil && *event != log.EventName {
changes["event"] = *event
log.EventName = *event
}
if description != nil && *description != log.Description {
changes["description"] = *description
log.Description = *description
}
if open != nil && *open != log.Open {
changes["open"] = *open
log.Open = *open
}
if len(changes) == 0 {
return log, nil
}
err := collection.UpdateId(log.ID, bson.M{"$set": changes})
if err != nil {
return models.Log{}, err
}
// There can be only one open log. TODO: Transaction
if changes["open"] != nil && *open {
query := bson.M{
"_id": bson.M{"$ne": log.ID},
"open": true,
"channel": log.ChannelName,
}
go collection.UpdateAll(query, bson.M{"$set": bson.M{"open": false}})
}
return log, nil
}

16
models/logs/find.go

@ -1,16 +0,0 @@
package logs
import (
"git.aiterp.net/rpdata/api/models"
"github.com/globalsign/mgo/bson"
)
// FindID finds a log either by it's ID or short ID.
func FindID(id string) (models.Log, error) {
return find(bson.M{
"$or": []bson.M{
bson.M{"_id": id},
bson.M{"shortId": id},
},
})
}

70
models/logs/list.go

@ -1,70 +0,0 @@
package logs
import (
"git.aiterp.net/rpdata/api/models"
"github.com/globalsign/mgo/bson"
)
// Filter for the List() function
type Filter struct {
Search *string
Characters *[]string
Channels *[]string
Events *[]string
Open *bool
Limit int
}
// List lists all logs
func List(filter *Filter) ([]models.Log, error) {
query := bson.M{}
limit := 0
if filter != nil {
// Run a text search
if filter.Search != nil {
searchResults, err := search(*filter.Search)
if err != nil {
return nil, err
}
// Posts always use shortId to refer to the log
query["shortId"] = bson.M{"$in": searchResults}
}
// Find logs including any of the specified events and channels
if filter.Channels != nil {
query["channel"] = bson.M{"$in": *filter.Channels}
}
if filter.Events != nil {
query["event"] = bson.M{"$in": *filter.Events}
}
// Find logs including all of the specified character IDs.
if filter.Characters != nil {
query["characterIds"] = bson.M{"$all": *filter.Characters}
}
// Limit to only open logs
if filter.Open != nil {
query["open"] = *filter.Open
}
// Set the limit from the filter
limit = filter.Limit
}
return list(query, limit)
}
func search(text string) ([]string, error) {
query := bson.M{
"$text": bson.M{"$search": text},
"logId": bson.M{"$ne": nil},
}
ids := make([]string, 0, 64)
err := postCollection.Find(query).Distinct("logId", &ids)
return ids, err
}

22
models/logs/remove.go

@ -1,22 +0,0 @@
package logs
import (
"errors"
"git.aiterp.net/rpdata/api/models"
"git.aiterp.net/rpdata/api/models/posts"
)
// Remove removes the log.
func Remove(log models.Log) (models.Log, error) {
err := collection.RemoveId(log.ID)
if err != nil {
return models.Log{}, err
}
if err := posts.RemoveAllInLog(log); err != nil {
return models.Log{}, errors.New("The log was removed, but its posts couldn't be: " + err.Error())
}
return log, nil
}

138
models/logs/update-characters.go

@ -1,138 +0,0 @@
package logs
import (
"errors"
"strings"
"time"
"git.aiterp.net/rpdata/api/internal/task"
"git.aiterp.net/rpdata/api/models"
"git.aiterp.net/rpdata/api/models/characters"
"git.aiterp.net/rpdata/api/models/posts"
"git.aiterp.net/rpdata/api/models/unknownnicks"
"github.com/globalsign/mgo/bson"
)
var updateTask = task.New(time.Second*60, RunFullUpdate)
// UpdateCharacters updates the characters for the given log.
func UpdateCharacters(log models.Log, unknowns map[string]int) (models.Log, error) {
posts, err := posts.List(&posts.Filter{LogID: &log.ShortID, Kind: []string{"action", "text", "chars"}, Limit: 0})
if err != nil {
return models.Log{}, err
}
counts := make(map[string]int)
added := make(map[string]bool)
removed := make(map[string]bool)
for _, post := range posts {
if post.Kind == "text" || post.Kind == "action" {
if strings.HasPrefix(post.Text, "(") || strings.Contains(post.Nick, "(") || strings.Contains(post.Nick, "[E]") || strings.HasSuffix(post.Nick, "|") {
continue
}
// Clean up the nick (remove possessive suffix, comma, formatting stuff)
if strings.HasSuffix(post.Nick, "'s") || strings.HasSuffix(post.Nick, "`s") {
post.Nick = post.Nick[:len(post.Nick)-2]
} else if strings.HasSuffix(post.Nick, "'") || strings.HasSuffix(post.Nick, "`") || strings.HasSuffix(post.Nick, ",") || strings.HasSuffix(post.Nick, "\x0f") {
post.Nick = post.Nick[:len(post.Nick)-1]
}
added[post.Nick] = true
counts[post.Nick]++
}
if post.Kind == "chars" {
tokens := strings.Fields(post.Text)
for _, token := range tokens {
if strings.HasPrefix(token, "-") {
removed[token[1:]] = true
} else {
added[strings.Replace(token, "+", "", 1)] = true
}
}
}
}
nicks := make([]string, 0, len(added))
for nick := range added {
if added[nick] && !removed[nick] {
nicks = append(nicks, nick)
}
}
characters, err := characters.List(&characters.Filter{Nicks: nicks})
if err != nil {
return models.Log{}, err
}
characterIDs := make([]string, len(characters))
for i, char := range characters {
if char.Name == "(Hidden)" {
continue
}
characterIDs[i] = char.ID
}
err = collection.UpdateId(log.ID, bson.M{"$set": bson.M{"characterIds": characterIDs}})
if err != nil {
return models.Log{}, err
}
log.CharacterIDs = characterIDs
if len(nicks) > 0 && unknowns != nil {
NickLoop:
for nick := range added {
if !added[nick] {
continue
}
for _, character := range characters {
if character.HasNick(nick) {
continue NickLoop
}
}
unknowns[nick] += counts[nick]
}
}
return log, nil
}
// RunFullUpdate runs a full update on all logs.
func RunFullUpdate() error {
iter := iter(bson.M{}, 0)
err := iter.Err()
if err != nil {
return err
}
unknowns := make(map[string]int, 256)
log := models.Log{}
for iter.Next(&log) {
_, err = UpdateCharacters(log, unknowns)
if err != nil {
return err
}
}
err = iter.Err()
if err != nil {
return err
}
err = unknownnicks.Update(unknowns)
if err != nil {
return errors.New("Failed to commit unknown nicks update: " + err.Error())
}
return nil
}
// ScheduleFullUpdate runs a full character update within the next 60 seconds.
func ScheduleFullUpdate() {
updateTask.Schedule()
}

5
services/channels.go

@ -66,11 +66,6 @@ func (s *ChannelService) Create(ctx context.Context, name string, logged, hub bo
} }
func (s *ChannelService) Ensure(ctx context.Context, name string) (*models.Channel, error) { func (s *ChannelService) Ensure(ctx context.Context, name string) (*models.Channel, error) {
err := auth.CheckPermission(ctx, "add", &models.Channel{})
if err != nil {
return nil, err
}
channel, err := s.channels.Find(ctx, name) channel, err := s.channels.Find(ctx, name)
if err != nil { if err != nil {
channel, err = s.channels.Insert(ctx, models.Channel{ channel, err = s.channels.Insert(ctx, models.Channel{

43
services/logs.go

@ -97,16 +97,16 @@ func (s *LogService) Import(ctx context.Context, importer models.LogImporter, da
results := make([]*models.Log, 0, 8) results := make([]*models.Log, 0, 8)
eventName := ""
if channel, err := channels.FindName(channelName); err != nil {
eventName = channel.EventName
}
_, err := s.channelService.Ensure(ctx, channelName) _, err := s.channelService.Ensure(ctx, channelName)
if err != nil { if err != nil {
return nil, err return nil, err
} }
eventName := ""
if channel, err := channels.FindName(channelName); err == nil {
eventName = channel.EventName
}
date = date.In(tz) date = date.In(tz)
switch importer { switch importer {
@ -134,6 +134,8 @@ func (s *LogService) Import(ctx context.Context, importer models.LogImporter, da
posts, err := s.posts.InsertMany(ctx, parsed.Posts...) posts, err := s.posts.InsertMany(ctx, parsed.Posts...)
if err != nil { if err != nil {
_ = s.logs.Delete(ctx, *log)
return nil, err return nil, err
} }
@ -163,6 +165,8 @@ func (s *LogService) Import(ctx context.Context, importer models.LogImporter, da
posts, err := s.posts.InsertMany(ctx, parsed.Posts...) posts, err := s.posts.InsertMany(ctx, parsed.Posts...)
if err != nil { if err != nil {
_ = s.logs.Delete(ctx, *log)
return nil, err return nil, err
} }
@ -204,13 +208,13 @@ func (s *LogService) AddPost(ctx context.Context, logId string, time time.Time,
return nil, errors.New("kind, nick and time must be non-empty") return nil, errors.New("kind, nick and time must be non-empty")
} }
log, err := s.logs.Find(ctx, logId)
l, err := s.logs.Find(ctx, logId)
if err != nil { if err != nil {
return nil, err return nil, err
} }
post := &models.Post{ post := &models.Post{
LogID: log.ShortID,
LogID: l.ShortID,
Kind: kind, Kind: kind,
Nick: nick, Nick: nick,
Text: text, Text: text,
@ -226,7 +230,12 @@ func (s *LogService) AddPost(ctx context.Context, logId string, time time.Time,
return nil, err return nil, err
} }
s.changeService.Submit(ctx, models.ChangeModelPost, "add", true, changekeys.Many(log, post), log, post)
err = s.refreshLogCharacters(ctx, *l, nil)
if err != nil {
log.Printf("Failed to update characters in log %s: %s", l.ID, err)
}
s.changeService.Submit(ctx, models.ChangeModelPost, "add", true, changekeys.Many(l, post), post)
return post, nil return post, nil
} }
@ -251,12 +260,17 @@ func (s *LogService) EditPost(ctx context.Context, id string, update models.Post
} }
go func() { go func() {
log, err := s.logs.Find(context.Background(), post.LogID)
l, err := s.logs.Find(context.Background(), post.LogID)
if err != nil { if err != nil {
return return
} }
s.changeService.Submit(ctx, models.ChangeModelPost, "edit", true, changekeys.Many(log, post), post)
err = s.refreshLogCharacters(ctx, *l, nil)
if err != nil {
log.Printf("Failed to update characters in log %s: %s", l.ID, err)
}
s.changeService.Submit(ctx, models.ChangeModelPost, "edit", true, changekeys.Many(l, post), post)
}() }()
return post, nil return post, nil
@ -313,12 +327,17 @@ func (s *LogService) DeletePost(ctx context.Context, id string) (*models.Post, e
} }
go func() { go func() {
log, err := s.logs.Find(context.Background(), post.LogID)
l, err := s.logs.Find(context.Background(), post.LogID)
if err != nil { if err != nil {
return return
} }
s.changeService.Submit(ctx, models.ChangeModelPost, "remove", true, changekeys.Many(log, post), post)
err = s.refreshLogCharacters(ctx, *l, nil)
if err != nil {
log.Printf("Failed to update characters in log %s: %s", l.ID, err)
}
s.changeService.Submit(ctx, models.ChangeModelPost, "remove", true, changekeys.Many(l, post), post)
}() }()
return post, nil return post, nil

Loading…
Cancel
Save