From 51b2c9bfe0658f097b055b275de652741030a735 Mon Sep 17 00:00:00 2001 From: Gisle Aune Date: Wed, 14 Aug 2019 11:28:02 +0200 Subject: [PATCH] Fixed log service bugs with importing and character list not updating. --- database/mongodb/posts.go | 2 +- models/logs/add.go | 55 ------------ models/logs/db.go | 48 ----------- models/logs/edit.go | 50 ----------- models/logs/find.go | 16 ---- models/logs/list.go | 70 ---------------- models/logs/remove.go | 22 ----- models/logs/update-characters.go | 138 ------------------------------- services/channels.go | 5 -- services/logs.go | 43 +++++++--- 10 files changed, 32 insertions(+), 417 deletions(-) delete mode 100644 models/logs/add.go delete mode 100644 models/logs/db.go delete mode 100644 models/logs/edit.go delete mode 100644 models/logs/find.go delete mode 100644 models/logs/list.go delete mode 100644 models/logs/remove.go delete mode 100644 models/logs/update-characters.go diff --git a/database/mongodb/posts.go b/database/mongodb/posts.go index 7201c61..5556231 100644 --- a/database/mongodb/posts.go +++ b/database/mongodb/posts.go @@ -130,7 +130,7 @@ func (r *postRepository) InsertMany(ctx context.Context, posts ...*models.Post) } logId := posts[0].LogID - for _, post := range posts[1:] { + for _, post := range posts { if post.LogID != logId { return nil, repositories.ErrParentMismatch } diff --git a/models/logs/add.go b/models/logs/add.go deleted file mode 100644 index bf077fb..0000000 --- a/models/logs/add.go +++ /dev/null @@ -1,55 +0,0 @@ -package logs - -import ( - "errors" - "strconv" - "time" - - "git.aiterp.net/rpdata/api/internal/counter" - "git.aiterp.net/rpdata/api/models" - "git.aiterp.net/rpdata/api/models/channels" - "github.com/globalsign/mgo/bson" -) - -// Add creates a new Log -func Add(date time.Time, channelName, title, eventName, description string, open bool) (models.Log, error) { - nextID, err := counter.Next("auto_increment", "Log") - if err != nil { - return models.Log{}, errors.New("Failed to allocate short ID: " + err.Error()) - } - - _, err = channels.Ensure(channelName, open) - if err != nil { - return models.Log{}, errors.New("Failed to ensure channel: " + err.Error()) - } - - log := models.Log{ - ID: makeLogID(date, channelName), - ShortID: "L" + strconv.Itoa(nextID), - Date: date, - ChannelName: channelName, - Title: title, - EventName: eventName, - Description: description, - Open: open, - CharacterIDs: nil, - } - - err = collection.Insert(log) - if err != nil { - return models.Log{}, err - } - - // There can be only one open log in the same channel. TODO: Transaction - if open { - query := bson.M{ - "_id": bson.M{"$ne": log.ID}, - "open": true, - "channel": log.ChannelName, - } - - go collection.UpdateAll(query, bson.M{"$set": bson.M{"open": false}}) - } - - return log, nil -} diff --git a/models/logs/db.go b/models/logs/db.go deleted file mode 100644 index 604a613..0000000 --- a/models/logs/db.go +++ /dev/null @@ -1,48 +0,0 @@ -package logs - -import ( - "fmt" - "time" - - "git.aiterp.net/rpdata/api/internal/store" - "git.aiterp.net/rpdata/api/models" - "github.com/globalsign/mgo" -) - -var collection *mgo.Collection -var postCollection *mgo.Collection - -func find(query interface{}) (models.Log, error) { - log := models.Log{} - err := collection.Find(query).One(&log) - if err != nil { - return models.Log{}, err - } - - return log, nil -} - -func list(query interface{}, limit int) ([]models.Log, error) { - logs := make([]models.Log, 0, 64) - err := collection.Find(query).Limit(limit).Sort("-date").All(&logs) - if err != nil { - return nil, err - } - - return logs, nil -} - -func iter(query interface{}, limit int) *mgo.Iter { - return collection.Find(query).Sort("-date").Limit(limit).Batch(8).Iter() -} - -func makeLogID(date time.Time, channel string) string { - return fmt.Sprintf("%s%03d_%s", date.UTC().Format("2006-01-02_150405"), (date.Nanosecond() / int(time.Millisecond/time.Nanosecond)), channel[1:]) -} - -func init() { - store.HandleInit(func(db *mgo.Database) { - collection = db.C("logbot3.logs") - postCollection = db.C("logbot3.posts") - }) -} diff --git a/models/logs/edit.go b/models/logs/edit.go deleted file mode 100644 index 076c6c8..0000000 --- a/models/logs/edit.go +++ /dev/null @@ -1,50 +0,0 @@ -package logs - -import ( - "git.aiterp.net/rpdata/api/models" - "github.com/globalsign/mgo/bson" -) - -// Edit sets a log's meta-data. -func Edit(log models.Log, title *string, event *string, description *string, open *bool) (models.Log, error) { - changes := bson.M{} - - if title != nil && *title != log.Title { - changes["title"] = *title - log.Title = *title - } - if event != nil && *event != log.EventName { - changes["event"] = *event - log.EventName = *event - } - if description != nil && *description != log.Description { - changes["description"] = *description - log.Description = *description - } - if open != nil && *open != log.Open { - changes["open"] = *open - log.Open = *open - } - - if len(changes) == 0 { - return log, nil - } - - err := collection.UpdateId(log.ID, bson.M{"$set": changes}) - if err != nil { - return models.Log{}, err - } - - // There can be only one open log. TODO: Transaction - if changes["open"] != nil && *open { - query := bson.M{ - "_id": bson.M{"$ne": log.ID}, - "open": true, - "channel": log.ChannelName, - } - - go collection.UpdateAll(query, bson.M{"$set": bson.M{"open": false}}) - } - - return log, nil -} diff --git a/models/logs/find.go b/models/logs/find.go deleted file mode 100644 index 9a40f0f..0000000 --- a/models/logs/find.go +++ /dev/null @@ -1,16 +0,0 @@ -package logs - -import ( - "git.aiterp.net/rpdata/api/models" - "github.com/globalsign/mgo/bson" -) - -// FindID finds a log either by it's ID or short ID. -func FindID(id string) (models.Log, error) { - return find(bson.M{ - "$or": []bson.M{ - bson.M{"_id": id}, - bson.M{"shortId": id}, - }, - }) -} diff --git a/models/logs/list.go b/models/logs/list.go deleted file mode 100644 index 4a3ccb6..0000000 --- a/models/logs/list.go +++ /dev/null @@ -1,70 +0,0 @@ -package logs - -import ( - "git.aiterp.net/rpdata/api/models" - "github.com/globalsign/mgo/bson" -) - -// Filter for the List() function -type Filter struct { - Search *string - Characters *[]string - Channels *[]string - Events *[]string - Open *bool - Limit int -} - -// List lists all logs -func List(filter *Filter) ([]models.Log, error) { - query := bson.M{} - limit := 0 - - if filter != nil { - // Run a text search - if filter.Search != nil { - searchResults, err := search(*filter.Search) - if err != nil { - return nil, err - } - - // Posts always use shortId to refer to the log - query["shortId"] = bson.M{"$in": searchResults} - } - - // Find logs including any of the specified events and channels - if filter.Channels != nil { - query["channel"] = bson.M{"$in": *filter.Channels} - } - if filter.Events != nil { - query["event"] = bson.M{"$in": *filter.Events} - } - - // Find logs including all of the specified character IDs. - if filter.Characters != nil { - query["characterIds"] = bson.M{"$all": *filter.Characters} - } - - // Limit to only open logs - if filter.Open != nil { - query["open"] = *filter.Open - } - - // Set the limit from the filter - limit = filter.Limit - } - - return list(query, limit) -} - -func search(text string) ([]string, error) { - query := bson.M{ - "$text": bson.M{"$search": text}, - "logId": bson.M{"$ne": nil}, - } - - ids := make([]string, 0, 64) - err := postCollection.Find(query).Distinct("logId", &ids) - - return ids, err -} diff --git a/models/logs/remove.go b/models/logs/remove.go deleted file mode 100644 index 0d68f98..0000000 --- a/models/logs/remove.go +++ /dev/null @@ -1,22 +0,0 @@ -package logs - -import ( - "errors" - - "git.aiterp.net/rpdata/api/models" - "git.aiterp.net/rpdata/api/models/posts" -) - -// Remove removes the log. -func Remove(log models.Log) (models.Log, error) { - err := collection.RemoveId(log.ID) - if err != nil { - return models.Log{}, err - } - - if err := posts.RemoveAllInLog(log); err != nil { - return models.Log{}, errors.New("The log was removed, but its posts couldn't be: " + err.Error()) - } - - return log, nil -} \ No newline at end of file diff --git a/models/logs/update-characters.go b/models/logs/update-characters.go deleted file mode 100644 index 8d8cfab..0000000 --- a/models/logs/update-characters.go +++ /dev/null @@ -1,138 +0,0 @@ -package logs - -import ( - "errors" - "strings" - "time" - - "git.aiterp.net/rpdata/api/internal/task" - "git.aiterp.net/rpdata/api/models" - "git.aiterp.net/rpdata/api/models/characters" - "git.aiterp.net/rpdata/api/models/posts" - "git.aiterp.net/rpdata/api/models/unknownnicks" - "github.com/globalsign/mgo/bson" -) - -var updateTask = task.New(time.Second*60, RunFullUpdate) - -// UpdateCharacters updates the characters for the given log. -func UpdateCharacters(log models.Log, unknowns map[string]int) (models.Log, error) { - posts, err := posts.List(&posts.Filter{LogID: &log.ShortID, Kind: []string{"action", "text", "chars"}, Limit: 0}) - if err != nil { - return models.Log{}, err - } - - counts := make(map[string]int) - added := make(map[string]bool) - removed := make(map[string]bool) - for _, post := range posts { - if post.Kind == "text" || post.Kind == "action" { - if strings.HasPrefix(post.Text, "(") || strings.Contains(post.Nick, "(") || strings.Contains(post.Nick, "[E]") || strings.HasSuffix(post.Nick, "|") { - continue - } - - // Clean up the nick (remove possessive suffix, comma, formatting stuff) - if strings.HasSuffix(post.Nick, "'s") || strings.HasSuffix(post.Nick, "`s") { - post.Nick = post.Nick[:len(post.Nick)-2] - } else if strings.HasSuffix(post.Nick, "'") || strings.HasSuffix(post.Nick, "`") || strings.HasSuffix(post.Nick, ",") || strings.HasSuffix(post.Nick, "\x0f") { - post.Nick = post.Nick[:len(post.Nick)-1] - } - - added[post.Nick] = true - counts[post.Nick]++ - } - if post.Kind == "chars" { - tokens := strings.Fields(post.Text) - for _, token := range tokens { - if strings.HasPrefix(token, "-") { - removed[token[1:]] = true - } else { - added[strings.Replace(token, "+", "", 1)] = true - } - } - } - } - - nicks := make([]string, 0, len(added)) - for nick := range added { - if added[nick] && !removed[nick] { - nicks = append(nicks, nick) - } - } - - characters, err := characters.List(&characters.Filter{Nicks: nicks}) - if err != nil { - return models.Log{}, err - } - - characterIDs := make([]string, len(characters)) - for i, char := range characters { - if char.Name == "(Hidden)" { - continue - } - - characterIDs[i] = char.ID - } - - err = collection.UpdateId(log.ID, bson.M{"$set": bson.M{"characterIds": characterIDs}}) - if err != nil { - return models.Log{}, err - } - - log.CharacterIDs = characterIDs - - if len(nicks) > 0 && unknowns != nil { - NickLoop: - for nick := range added { - if !added[nick] { - continue - } - - for _, character := range characters { - if character.HasNick(nick) { - continue NickLoop - } - } - - unknowns[nick] += counts[nick] - } - } - - return log, nil -} - -// RunFullUpdate runs a full update on all logs. -func RunFullUpdate() error { - iter := iter(bson.M{}, 0) - err := iter.Err() - if err != nil { - return err - } - - unknowns := make(map[string]int, 256) - - log := models.Log{} - for iter.Next(&log) { - _, err = UpdateCharacters(log, unknowns) - if err != nil { - return err - } - } - - err = iter.Err() - if err != nil { - return err - } - - err = unknownnicks.Update(unknowns) - if err != nil { - return errors.New("Failed to commit unknown nicks update: " + err.Error()) - } - - return nil -} - -// ScheduleFullUpdate runs a full character update within the next 60 seconds. -func ScheduleFullUpdate() { - updateTask.Schedule() -} diff --git a/services/channels.go b/services/channels.go index b792c4a..b1d11b4 100644 --- a/services/channels.go +++ b/services/channels.go @@ -66,11 +66,6 @@ func (s *ChannelService) Create(ctx context.Context, name string, logged, hub bo } func (s *ChannelService) Ensure(ctx context.Context, name string) (*models.Channel, error) { - err := auth.CheckPermission(ctx, "add", &models.Channel{}) - if err != nil { - return nil, err - } - channel, err := s.channels.Find(ctx, name) if err != nil { channel, err = s.channels.Insert(ctx, models.Channel{ diff --git a/services/logs.go b/services/logs.go index c0a7882..002a901 100644 --- a/services/logs.go +++ b/services/logs.go @@ -97,16 +97,16 @@ func (s *LogService) Import(ctx context.Context, importer models.LogImporter, da results := make([]*models.Log, 0, 8) - eventName := "" - if channel, err := channels.FindName(channelName); err != nil { - eventName = channel.EventName - } - _, err := s.channelService.Ensure(ctx, channelName) if err != nil { return nil, err } + eventName := "" + if channel, err := channels.FindName(channelName); err == nil { + eventName = channel.EventName + } + date = date.In(tz) switch importer { @@ -134,6 +134,8 @@ func (s *LogService) Import(ctx context.Context, importer models.LogImporter, da posts, err := s.posts.InsertMany(ctx, parsed.Posts...) if err != nil { + _ = s.logs.Delete(ctx, *log) + return nil, err } @@ -163,6 +165,8 @@ func (s *LogService) Import(ctx context.Context, importer models.LogImporter, da posts, err := s.posts.InsertMany(ctx, parsed.Posts...) if err != nil { + _ = s.logs.Delete(ctx, *log) + return nil, err } @@ -204,13 +208,13 @@ func (s *LogService) AddPost(ctx context.Context, logId string, time time.Time, return nil, errors.New("kind, nick and time must be non-empty") } - log, err := s.logs.Find(ctx, logId) + l, err := s.logs.Find(ctx, logId) if err != nil { return nil, err } post := &models.Post{ - LogID: log.ShortID, + LogID: l.ShortID, Kind: kind, Nick: nick, Text: text, @@ -226,7 +230,12 @@ func (s *LogService) AddPost(ctx context.Context, logId string, time time.Time, return nil, err } - s.changeService.Submit(ctx, models.ChangeModelPost, "add", true, changekeys.Many(log, post), log, post) + err = s.refreshLogCharacters(ctx, *l, nil) + if err != nil { + log.Printf("Failed to update characters in log %s: %s", l.ID, err) + } + + s.changeService.Submit(ctx, models.ChangeModelPost, "add", true, changekeys.Many(l, post), post) return post, nil } @@ -251,12 +260,17 @@ func (s *LogService) EditPost(ctx context.Context, id string, update models.Post } go func() { - log, err := s.logs.Find(context.Background(), post.LogID) + l, err := s.logs.Find(context.Background(), post.LogID) if err != nil { return } - s.changeService.Submit(ctx, models.ChangeModelPost, "edit", true, changekeys.Many(log, post), post) + err = s.refreshLogCharacters(ctx, *l, nil) + if err != nil { + log.Printf("Failed to update characters in log %s: %s", l.ID, err) + } + + s.changeService.Submit(ctx, models.ChangeModelPost, "edit", true, changekeys.Many(l, post), post) }() return post, nil @@ -313,12 +327,17 @@ func (s *LogService) DeletePost(ctx context.Context, id string) (*models.Post, e } go func() { - log, err := s.logs.Find(context.Background(), post.LogID) + l, err := s.logs.Find(context.Background(), post.LogID) if err != nil { return } - s.changeService.Submit(ctx, models.ChangeModelPost, "remove", true, changekeys.Many(log, post), post) + err = s.refreshLogCharacters(ctx, *l, nil) + if err != nil { + log.Printf("Failed to update characters in log %s: %s", l.ID, err) + } + + s.changeService.Submit(ctx, models.ChangeModelPost, "remove", true, changekeys.Many(l, post), post) }() return post, nil