rpdata
/
api
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
Browse Source

Fixed log service bugs with importing and character list not updating.
the build was successful
Details

thegreatrefactor
Gisle Aune 5 years ago
parent
c0687d76ad
commit
51b2c9bfe0
10 changed files with 32 additions and 417 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      database/mongodb/posts.go
  2. 55
      models/logs/add.go
  3. 48
      models/logs/db.go
  4. 50
      models/logs/edit.go
  5. 16
      models/logs/find.go
  6. 70
      models/logs/list.go
  7. 22
      models/logs/remove.go
  8. 138
      models/logs/update-characters.go
  9. 5
      services/channels.go
  10. 43
      services/logs.go

2
database/mongodb/posts.go
View File

@ -130,7 +130,7 @@ func (r *postRepository) InsertMany(ctx context.Context, posts ...*models.Post)
} }
logId := posts[0].LogID logId := posts[0].LogID
for _, post := range posts[1:] {
for _, post := range posts {
if post.LogID != logId { if post.LogID != logId {
return nil, repositories.ErrParentMismatch return nil, repositories.ErrParentMismatch
} }

55
models/logs/add.go
View File

@ -1,55 +0,0 @@
package logs
import (
"errors"
"strconv"
"time"
"git.aiterp.net/rpdata/api/internal/counter"
"git.aiterp.net/rpdata/api/models"
"git.aiterp.net/rpdata/api/models/channels"
"github.com/globalsign/mgo/bson"
)
// Add creates a new Log
func Add(date time.Time, channelName, title, eventName, description string, open bool) (models.Log, error) {
nextID, err := counter.Next("auto_increment", "Log")
if err != nil {
return models.Log{}, errors.New("Failed to allocate short ID: " + err.Error())
}
_, err = channels.Ensure(channelName, open)
if err != nil {
return models.Log{}, errors.New("Failed to ensure channel: " + err.Error())
}
log := models.Log{
ID: makeLogID(date, channelName),
ShortID: "L" + strconv.Itoa(nextID),
Date: date,
ChannelName: channelName,
Title: title,
EventName: eventName,
Description: description,
Open: open,
CharacterIDs: nil,
}
err = collection.Insert(log)
if err != nil {
return models.Log{}, err
}
// There can be only one open log in the same channel. TODO: Transaction
if open {
query := bson.M{
"_id": bson.M{"$ne": log.ID},
"open": true,
"channel": log.ChannelName,
}
go collection.UpdateAll(query, bson.M{"$set": bson.M{"open": false}})
}
return log, nil
}

48
models/logs/db.go
View File

@ -1,48 +0,0 @@
package logs
import (
"fmt"
"time"
"git.aiterp.net/rpdata/api/internal/store"
"git.aiterp.net/rpdata/api/models"
"github.com/globalsign/mgo"
)
var collection *mgo.Collection
var postCollection *mgo.Collection
func find(query interface{}) (models.Log, error) {
log := models.Log{}
err := collection.Find(query).One(&log)
if err != nil {
return models.Log{}, err
}
return log, nil
}
func list(query interface{}, limit int) ([]models.Log, error) {
logs := make([]models.Log, 0, 64)
err := collection.Find(query).Limit(limit).Sort("-date").All(&logs)
if err != nil {
return nil, err
}
return logs, nil
}
func iter(query interface{}, limit int) *mgo.Iter {
return collection.Find(query).Sort("-date").Limit(limit).Batch(8).Iter()
}
func makeLogID(date time.Time, channel string) string {
return fmt.Sprintf("%s%03d_%s", date.UTC().Format("2006-01-02_150405"), (date.Nanosecond() / int(time.Millisecond/time.Nanosecond)), channel[1:])
}
func init() {
store.HandleInit(func(db *mgo.Database) {
collection = db.C("logbot3.logs")
postCollection = db.C("logbot3.posts")
})
}

50
models/logs/edit.go
View File

@ -1,50 +0,0 @@
package logs
import (
"git.aiterp.net/rpdata/api/models"
"github.com/globalsign/mgo/bson"
)
// Edit sets a log's meta-data.
func Edit(log models.Log, title *string, event *string, description *string, open *bool) (models.Log, error) {
changes := bson.M{}
if title != nil && *title != log.Title {
changes["title"] = *title
log.Title = *title
}
if event != nil && *event != log.EventName {
changes["event"] = *event
log.EventName = *event
}
if description != nil && *description != log.Description {
changes["description"] = *description
log.Description = *description
}
if open != nil && *open != log.Open {
changes["open"] = *open
log.Open = *open
}
if len(changes) == 0 {
return log, nil
}
err := collection.UpdateId(log.ID, bson.M{"$set": changes})
if err != nil {
return models.Log{}, err
}
// There can be only one open log. TODO: Transaction
if changes["open"] != nil && *open {
query := bson.M{
"_id": bson.M{"$ne": log.ID},
"open": true,
"channel": log.ChannelName,
}
go collection.UpdateAll(query, bson.M{"$set": bson.M{"open": false}})
}
return log, nil
}

16
models/logs/find.go
View File

@ -1,16 +0,0 @@
package logs
import (
"git.aiterp.net/rpdata/api/models"
"github.com/globalsign/mgo/bson"
)
// FindID finds a log either by it's ID or short ID.
func FindID(id string) (models.Log, error) {
return find(bson.M{
"$or": []bson.M{
bson.M{"_id": id},
bson.M{"shortId": id},
},
})
}

70
models/logs/list.go
View File

@ -1,70 +0,0 @@
package logs
import (
"git.aiterp.net/rpdata/api/models"
"github.com/globalsign/mgo/bson"
)
// Filter for the List() function
type Filter struct {
Search *string
Characters *[]string
Channels *[]string
Events *[]string
Open *bool
Limit int
}
// List lists all logs
func List(filter *Filter) ([]models.Log, error) {
query := bson.M{}
limit := 0
if filter != nil {
// Run a text search
if filter.Search != nil {
searchResults, err := search(*filter.Search)
if err != nil {
return nil, err
}
// Posts always use shortId to refer to the log
query["shortId"] = bson.M{"$in": searchResults}
}
// Find logs including any of the specified events and channels
if filter.Channels != nil {
query["channel"] = bson.M{"$in": *filter.Channels}
}
if filter.Events != nil {
query["event"] = bson.M{"$in": *filter.Events}
}
// Find logs including all of the specified character IDs.
if filter.Characters != nil {
query["characterIds"] = bson.M{"$all": *filter.Characters}
}
// Limit to only open logs
if filter.Open != nil {
query["open"] = *filter.Open
}
// Set the limit from the filter
limit = filter.Limit
}
return list(query, limit)
}
func search(text string) ([]string, error) {
query := bson.M{
"$text": bson.M{"$search": text},
"logId": bson.M{"$ne": nil},
}
ids := make([]string, 0, 64)
err := postCollection.Find(query).Distinct("logId", &ids)
return ids, err
}

22
models/logs/remove.go
View File

@ -1,22 +0,0 @@
package logs
import (
"errors"
"git.aiterp.net/rpdata/api/models"
"git.aiterp.net/rpdata/api/models/posts"
)
// Remove removes the log.
func Remove(log models.Log) (models.Log, error) {
err := collection.RemoveId(log.ID)
if err != nil {
return models.Log{}, err
}
if err := posts.RemoveAllInLog(log); err != nil {
return models.Log{}, errors.New("The log was removed, but its posts couldn't be: " + err.Error())
}
return log, nil
}

138
models/logs/update-characters.go
View File

@ -1,138 +0,0 @@
package logs
import (
"errors"
"strings"
"time"
"git.aiterp.net/rpdata/api/internal/task"
"git.aiterp.net/rpdata/api/models"
"git.aiterp.net/rpdata/api/models/characters"
"git.aiterp.net/rpdata/api/models/posts"
"git.aiterp.net/rpdata/api/models/unknownnicks"
"github.com/globalsign/mgo/bson"
)
var updateTask = task.New(time.Second*60, RunFullUpdate)
// UpdateCharacters updates the characters for the given log.
func UpdateCharacters(log models.Log, unknowns map[string]int) (models.Log, error) {
posts, err := posts.List(&posts.Filter{LogID: &log.ShortID, Kind: []string{"action", "text", "chars"}, Limit: 0})
if err != nil {
return models.Log{}, err
}
counts := make(map[string]int)
added := make(map[string]bool)
removed := make(map[string]bool)
for _, post := range posts {
if post.Kind == "text" || post.Kind == "action" {
if strings.HasPrefix(post.Text, "(") || strings.Contains(post.Nick, "(") || strings.Contains(post.Nick, "[E]") || strings.HasSuffix(post.Nick, "|") {
continue
}
// Clean up the nick (remove possessive suffix, comma, formatting stuff)
if strings.HasSuffix(post.Nick, "'s") || strings.HasSuffix(post.Nick, "`s") {
post.Nick = post.Nick[:len(post.Nick)-2]
} else if strings.HasSuffix(post.Nick, "'") || strings.HasSuffix(post.Nick, "`") || strings.HasSuffix(post.Nick, ",") || strings.HasSuffix(post.Nick, "\x0f") {
post.Nick = post.Nick[:len(post.Nick)-1]
}
added[post.Nick] = true
counts[post.Nick]++
}
if post.Kind == "chars" {
tokens := strings.Fields(post.Text)
for _, token := range tokens {
if strings.HasPrefix(token, "-") {
removed[token[1:]] = true
} else {
added[strings.Replace(token, "+", "", 1)] = true
}
}
}
}
nicks := make([]string, 0, len(added))
for nick := range added {
if added[nick] && !removed[nick] {
nicks = append(nicks, nick)
}
}
characters, err := characters.List(&characters.Filter{Nicks: nicks})
if err != nil {
return models.Log{}, err
}
characterIDs := make([]string, len(characters))
for i, char := range characters {
if char.Name == "(Hidden)" {
continue
}
characterIDs[i] = char.ID
}
err = collection.UpdateId(log.ID, bson.M{"$set": bson.M{"characterIds": characterIDs}})
if err != nil {
return models.Log{}, err
}
log.CharacterIDs = characterIDs
if len(nicks) > 0 && unknowns != nil {
NickLoop:
for nick := range added {
if !added[nick] {
continue
}
for _, character := range characters {
if character.HasNick(nick) {
continue NickLoop
}
}
unknowns[nick] += counts[nick]
}
}
return log, nil
}
// RunFullUpdate runs a full update on all logs.
func RunFullUpdate() error {
iter := iter(bson.M{}, 0)
err := iter.Err()
if err != nil {
return err
}
unknowns := make(map[string]int, 256)
log := models.Log{}
for iter.Next(&log) {
_, err = UpdateCharacters(log, unknowns)
if err != nil {
return err
}
}
err = iter.Err()
if err != nil {
return err
}
err = unknownnicks.Update(unknowns)
if err != nil {
return errors.New("Failed to commit unknown nicks update: " + err.Error())
}
return nil
}
// ScheduleFullUpdate runs a full character update within the next 60 seconds.
func ScheduleFullUpdate() {
updateTask.Schedule()
}

5
services/channels.go
View File

@ -66,11 +66,6 @@ func (s *ChannelService) Create(ctx context.Context, name string, logged, hub bo
} }
func (s *ChannelService) Ensure(ctx context.Context, name string) (*models.Channel, error) { func (s *ChannelService) Ensure(ctx context.Context, name string) (*models.Channel, error) {
err := auth.CheckPermission(ctx, "add", &models.Channel{})
if err != nil {
return nil, err
}
channel, err := s.channels.Find(ctx, name) channel, err := s.channels.Find(ctx, name)
if err != nil { if err != nil {
channel, err = s.channels.Insert(ctx, models.Channel{ channel, err = s.channels.Insert(ctx, models.Channel{

43
services/logs.go
View File

@ -97,16 +97,16 @@ func (s *LogService) Import(ctx context.Context, importer models.LogImporter, da
results := make([]*models.Log, 0, 8) results := make([]*models.Log, 0, 8)
eventName := ""
if channel, err := channels.FindName(channelName); err != nil {
eventName = channel.EventName
}
_, err := s.channelService.Ensure(ctx, channelName) _, err := s.channelService.Ensure(ctx, channelName)
if err != nil { if err != nil {
return nil, err return nil, err
} }
eventName := ""
if channel, err := channels.FindName(channelName); err == nil {
eventName = channel.EventName
}
date = date.In(tz) date = date.In(tz)
switch importer { switch importer {
@ -134,6 +134,8 @@ func (s *LogService) Import(ctx context.Context, importer models.LogImporter, da
posts, err := s.posts.InsertMany(ctx, parsed.Posts...) posts, err := s.posts.InsertMany(ctx, parsed.Posts...)
if err != nil { if err != nil {
_ = s.logs.Delete(ctx, *log)
return nil, err return nil, err
} }
@ -163,6 +165,8 @@ func (s *LogService) Import(ctx context.Context, importer models.LogImporter, da
posts, err := s.posts.InsertMany(ctx, parsed.Posts...) posts, err := s.posts.InsertMany(ctx, parsed.Posts...)
if err != nil { if err != nil {
_ = s.logs.Delete(ctx, *log)
return nil, err return nil, err
} }
@ -204,13 +208,13 @@ func (s *LogService) AddPost(ctx context.Context, logId string, time time.Time,
return nil, errors.New("kind, nick and time must be non-empty") return nil, errors.New("kind, nick and time must be non-empty")
} }
log, err := s.logs.Find(ctx, logId)
l, err := s.logs.Find(ctx, logId)
if err != nil { if err != nil {
return nil, err return nil, err
} }
post := &models.Post{ post := &models.Post{
LogID: log.ShortID,
LogID: l.ShortID,
Kind: kind, Kind: kind,
Nick: nick, Nick: nick,
Text: text, Text: text,
@ -226,7 +230,12 @@ func (s *LogService) AddPost(ctx context.Context, logId string, time time.Time,
return nil, err return nil, err
} }
s.changeService.Submit(ctx, models.ChangeModelPost, "add", true, changekeys.Many(log, post), log, post)
err = s.refreshLogCharacters(ctx, *l, nil)
if err != nil {
log.Printf("Failed to update characters in log %s: %s", l.ID, err)
}
s.changeService.Submit(ctx, models.ChangeModelPost, "add", true, changekeys.Many(l, post), post)
return post, nil return post, nil
} }
@ -251,12 +260,17 @@ func (s *LogService) EditPost(ctx context.Context, id string, update models.Post
} }
go func() { go func() {
log, err := s.logs.Find(context.Background(), post.LogID)
l, err := s.logs.Find(context.Background(), post.LogID)
if err != nil { if err != nil {
return return
} }
s.changeService.Submit(ctx, models.ChangeModelPost, "edit", true, changekeys.Many(log, post), post)
err = s.refreshLogCharacters(ctx, *l, nil)
if err != nil {
log.Printf("Failed to update characters in log %s: %s", l.ID, err)
}
s.changeService.Submit(ctx, models.ChangeModelPost, "edit", true, changekeys.Many(l, post), post)
}() }()
return post, nil return post, nil
@ -313,12 +327,17 @@ func (s *LogService) DeletePost(ctx context.Context, id string) (*models.Post, e
} }
go func() { go func() {
log, err := s.logs.Find(context.Background(), post.LogID)
l, err := s.logs.Find(context.Background(), post.LogID)
if err != nil { if err != nil {
return return
} }
s.changeService.Submit(ctx, models.ChangeModelPost, "remove", true, changekeys.Many(log, post), post)
err = s.refreshLogCharacters(ctx, *l, nil)
if err != nil {
log.Printf("Failed to update characters in log %s: %s", l.ID, err)
}
s.changeService.Submit(ctx, models.ChangeModelPost, "remove", true, changekeys.Many(l, post), post)
}() }()
return post, nil return post, nil

Write Preview
Loading…
Cancel
Save