rpdata/api
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Moved unknown nicks to log service
the build was successful
Details

Browse Source
This commit is contained in:
gisle
2019-08-26 20:23:36 +02:00
parent 3b10c4d9b1
commit 0e70e37038
7 changed files with 62 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files
graph2/resolvers/unknownnicks.go
+6 -14
View File
@@ -5,28 +5,20 @@ import (
"git.aiterp.net/rpdata/api/graph2/graphcore"
"git.aiterp.net/rpdata/api/models"
"git.aiterp.net/rpdata/api/models/unknownnicks"
)
/// Queries
func (r *queryResolver) UnknownNicks(ctx context.Context, filter *graphcore.UnknownNicksFilter) ([]*models.UnknownNick, error) {
limit := 100
limit := 20
if filter != nil {
if filter.Limit > 0 {
limit = filter.Limit
}
limit = filter.Limit
}
nicks, err := unknownnicks.List(limit)
if err != nil {
return nil, err
nicks := r.s.Logs.ListUnknownNicks()
if limit > 0 && len(nicks) > limit {
nicks = nicks[:limit]
}
nicks2 := make([]*models.UnknownNick, len(nicks))
for i := range nicks {
nicks2[i] = &nicks[i]
}
return nicks2, nil
return nicks, nil
}
models/unknownnicks/db.go
-37
View File
@@ -1,37 +0,0 @@
package unknownnicks
import (
"log"
"git.aiterp.net/rpdata/api/internal/store"
"git.aiterp.net/rpdata/api/models"
"github.com/globalsign/mgo"
)
var collection *mgo.Collection
func list(query interface{}, limit int) ([]models.UnknownNick, error) {
size := 64
if limit > 0 {
size = limit
}
posts := make([]models.UnknownNick, 0, size)
err := collection.Find(query).Limit(limit).Sort("-score").All(&posts)
if err != nil {
return nil, err
}
return posts, nil
}
func init() {
store.HandleInit(func(db *mgo.Database) {
collection = db.C("logbot3.unkownnicks")
err := collection.EnsureIndexKey("weight")
if err != nil {
log.Fatalln("init logbot3.unkownnicks:", err)
}
})
}
models/unknownnicks/list.go
-10
View File
@@ -1,10 +0,0 @@
package unknownnicks
import (
"git.aiterp.net/rpdata/api/models"
)
// List lists the top N unknown nicks.
func List(limit int) ([]models.UnknownNick, error) {
return list(nil, limit)
}
models/unknownnicks/remove.go
-1
View File
@@ -1 +0,0 @@
package unknownnicks
models/unknownnicks/update.go
-22
View File
@@ -1,22 +0,0 @@
package unknownnicks
import (
"github.com/globalsign/mgo/bson"
)
// Update updates the map, replacing it with the scores provided here.
func Update(updateMap map[string]int) error {
_, err := collection.RemoveAll(bson.M{})
if err != nil {
return err
}
for nick, score := range updateMap {
_, err := collection.UpsertId(nick, bson.M{"$set": bson.M{"score": score}})
if err != nil {
return err
}
}
return nil
}
services/logs.go
+54 -11
View File
@@ -11,7 +11,9 @@ import (
"git.aiterp.net/rpdata/api/services/parsers"
"golang.org/x/sync/errgroup"
"log"
"sort"
"strings"
"sync"
"time"
)
@@ -21,6 +23,9 @@ type LogService struct {
changeService *ChangeService
channelService *ChannelService
characterService *CharacterService
unknownNicks map[string]int
unknownNicksMutex sync.Mutex
}
func (s *LogService) Find(ctx context.Context, id string) (*models.Log, error) {
@@ -56,6 +61,24 @@ func (s *LogService) ListPosts(ctx context.Context, filter *models.PostFilter) (
return s.posts.List(ctx, *filter)
}
func (s *LogService) ListUnknownNicks() []*models.UnknownNick {
s.unknownNicksMutex.Lock()
nicks := make([]*models.UnknownNick, 0, len(s.unknownNicks))
for nick, score := range s.unknownNicks {
nicks = append(nicks, &models.UnknownNick{
Nick: nick,
Score: score,
})
}
s.unknownNicksMutex.Unlock()
sort.Slice(nicks, func(i, j int) bool {
return nicks[i].Score > nicks[j].Score
})
return nicks
}
func (s *LogService) Create(ctx context.Context, title, description, channelName, eventName string, open bool) (*models.Log, error) {
if channelName == "" {
return nil, errors.New("channel name cannot be empty")
@@ -280,8 +303,8 @@ func (s *LogService) SplitLog(ctx context.Context, logId string, startPostId str
s.changeService.Submit(ctx, models.ChangeModelPost, "add", true, changekeys.Many(newLog, newPosts), newPosts)
// Refresh character lists.
_, _ = s.refreshLogCharacters(ctx, *l, nil)
newLog2, err := s.refreshLogCharacters(ctx, *newLog, nil)
_, _ = s.refreshLogCharacters(ctx, *l, nil, false)
newLog2, err := s.refreshLogCharacters(ctx, *newLog, nil, false)
if err == nil {
newLog = newLog2
}
@@ -339,7 +362,7 @@ func (s *LogService) MergeLogs(ctx context.Context, targetID string, sourceID st
}
// Refresh characters
target2, err := s.refreshLogCharacters(ctx, *target, nil)
target2, err := s.refreshLogCharacters(ctx, *target, nil, false)
if err != nil {
log.Printf("Failed to update characters in log %s: %s", target.ID, err)
} else {
@@ -379,7 +402,7 @@ func (s *LogService) AddPost(ctx context.Context, logId string, time time.Time,
return nil, err
}
l2, err := s.refreshLogCharacters(ctx, *l, nil)
l2, err := s.refreshLogCharacters(ctx, *l, nil, false)
if err != nil {
log.Printf("Failed to update characters in log %s: %s", l.ID, err)
} else {
@@ -416,7 +439,7 @@ func (s *LogService) EditPost(ctx context.Context, id string, update models.Post
return
}
_, err = s.refreshLogCharacters(ctx, *l, nil)
_, err = s.refreshLogCharacters(ctx, *l, nil, false)
if err != nil {
log.Printf("Failed to update characters in log %s: %s", l.ID, err)
}
@@ -483,7 +506,7 @@ func (s *LogService) DeletePost(ctx context.Context, id string) (*models.Post, e
return
}
_, err = s.refreshLogCharacters(ctx, *l, nil)
_, err = s.refreshLogCharacters(ctx, *l, nil, false)
if err != nil {
log.Printf("Failed to update characters in log %s: %s", l.ID, err)
}
@@ -638,12 +661,18 @@ func (s *LogService) RefreshAllLogCharacters(ctx context.Context) error {
}
characterMap := s.makeCharacterMap(characters)
s.unknownNicksMutex.Lock()
for key := range s.unknownNicks {
delete(s.unknownNicks, key)
}
s.unknownNicksMutex.Unlock()
eg := errgroup.Group{}
for i := range logs {
l := logs[i]
eg.Go(func() error {
_, err := s.refreshLogCharacters(ctx, *l, characterMap)
_, err := s.refreshLogCharacters(ctx, *l, characterMap, true)
return err
})
}
@@ -652,16 +681,20 @@ func (s *LogService) RefreshAllLogCharacters(ctx context.Context) error {
return err
}
log.Printf("Full log character refresh complete; nicks: %d, logs: %d, duration: %s", len(characterMap), len(logs), time.Since(start))
s.unknownNicksMutex.Lock()
unknownCount := len(s.unknownNicks)
s.unknownNicksMutex.Unlock()
log.Printf("Full log character refresh complete; nicks: %d, unknowns: %d, logs: %d, duration: %s", len(characterMap), unknownCount, len(logs), time.Since(start))
return nil
}
func (s *LogService) RefreshLogCharacters(ctx context.Context, log models.Log) (*models.Log, error) {
return s.refreshLogCharacters(ctx, log, nil)
return s.refreshLogCharacters(ctx, log, nil, false)
}
func (s *LogService) refreshLogCharacters(ctx context.Context, log models.Log, characterMap map[string]*models.Character) (*models.Log, error) {
func (s *LogService) refreshLogCharacters(ctx context.Context, log models.Log, characterMap map[string]*models.Character, useUnknownNicks bool) (*models.Log, error) {
posts, err := s.ListPosts(ctx, &models.PostFilter{LogID: &log.ShortID})
if err != nil {
return &log, nil
@@ -718,9 +751,19 @@ func (s *LogService) refreshLogCharacters(ctx context.Context, log models.Log, c
for key := range added {
delete(added, key)
}
unknowned := make(map[string]bool)
for _, nick := range nicks {
character := characterMap[nick]
if character == nil || added[character.ID] {
if character == nil {
if useUnknownNicks && !unknowned[nick] {
unknowned[nick] = true
s.unknownNicksMutex.Lock()
s.unknownNicks[nick]++
s.unknownNicksMutex.Unlock()
}
continue
} else if added[character.ID] {
continue
}
added[character.ID] = true
services/services.go
+2
View File
@@ -38,6 +38,8 @@ func NewBundle(db database.Database) *Bundle {
changeService: bundle.Changes,
channelService: bundle.Channels,
characterService: bundle.Characters,
unknownNicks: make(map[string]int, 512),
}
bundle.Stories = &StoryService{
stories: db.Stories(),