diff --git a/graph2/resolvers/unknownnicks.go b/graph2/resolvers/unknownnicks.go index 5f2c4a8..3abbfd8 100644 --- a/graph2/resolvers/unknownnicks.go +++ b/graph2/resolvers/unknownnicks.go @@ -5,28 +5,20 @@ import ( "git.aiterp.net/rpdata/api/graph2/graphcore" "git.aiterp.net/rpdata/api/models" - "git.aiterp.net/rpdata/api/models/unknownnicks" ) /// Queries func (r *queryResolver) UnknownNicks(ctx context.Context, filter *graphcore.UnknownNicksFilter) ([]*models.UnknownNick, error) { - limit := 100 + limit := 20 if filter != nil { - if filter.Limit > 0 { - limit = filter.Limit - } + limit = filter.Limit } - nicks, err := unknownnicks.List(limit) - if err != nil { - return nil, err + nicks := r.s.Logs.ListUnknownNicks() + if limit > 0 && len(nicks) > limit { + nicks = nicks[:limit] } - nicks2 := make([]*models.UnknownNick, len(nicks)) - for i := range nicks { - nicks2[i] = &nicks[i] - } - - return nicks2, nil + return nicks, nil } diff --git a/models/unknownnicks/db.go b/models/unknownnicks/db.go deleted file mode 100644 index 6502992..0000000 --- a/models/unknownnicks/db.go +++ /dev/null @@ -1,37 +0,0 @@ -package unknownnicks - -import ( - "log" - - "git.aiterp.net/rpdata/api/internal/store" - "git.aiterp.net/rpdata/api/models" - "github.com/globalsign/mgo" -) - -var collection *mgo.Collection - -func list(query interface{}, limit int) ([]models.UnknownNick, error) { - size := 64 - if limit > 0 { - size = limit - } - posts := make([]models.UnknownNick, 0, size) - - err := collection.Find(query).Limit(limit).Sort("-score").All(&posts) - if err != nil { - return nil, err - } - - return posts, nil -} - -func init() { - store.HandleInit(func(db *mgo.Database) { - collection = db.C("logbot3.unkownnicks") - - err := collection.EnsureIndexKey("weight") - if err != nil { - log.Fatalln("init logbot3.unkownnicks:", err) - } - }) -} diff --git a/models/unknownnicks/list.go b/models/unknownnicks/list.go deleted file mode 100644 index f261ff1..0000000 --- a/models/unknownnicks/list.go +++ /dev/null @@ -1,10 +0,0 @@ -package unknownnicks - -import ( - "git.aiterp.net/rpdata/api/models" -) - -// List lists the top N unknown nicks. -func List(limit int) ([]models.UnknownNick, error) { - return list(nil, limit) -} diff --git a/models/unknownnicks/remove.go b/models/unknownnicks/remove.go deleted file mode 100644 index a9c3894..0000000 --- a/models/unknownnicks/remove.go +++ /dev/null @@ -1 +0,0 @@ -package unknownnicks diff --git a/models/unknownnicks/update.go b/models/unknownnicks/update.go deleted file mode 100644 index 67deafe..0000000 --- a/models/unknownnicks/update.go +++ /dev/null @@ -1,22 +0,0 @@ -package unknownnicks - -import ( - "github.com/globalsign/mgo/bson" -) - -// Update updates the map, replacing it with the scores provided here. -func Update(updateMap map[string]int) error { - _, err := collection.RemoveAll(bson.M{}) - if err != nil { - return err - } - - for nick, score := range updateMap { - _, err := collection.UpsertId(nick, bson.M{"$set": bson.M{"score": score}}) - if err != nil { - return err - } - } - - return nil -} diff --git a/services/logs.go b/services/logs.go index 485517f..e60953f 100644 --- a/services/logs.go +++ b/services/logs.go @@ -11,7 +11,9 @@ import ( "git.aiterp.net/rpdata/api/services/parsers" "golang.org/x/sync/errgroup" "log" + "sort" "strings" + "sync" "time" ) @@ -21,6 +23,9 @@ type LogService struct { changeService *ChangeService channelService *ChannelService characterService *CharacterService + + unknownNicks map[string]int + unknownNicksMutex sync.Mutex } func (s *LogService) Find(ctx context.Context, id string) (*models.Log, error) { @@ -56,6 +61,24 @@ func (s *LogService) ListPosts(ctx context.Context, filter *models.PostFilter) ( return s.posts.List(ctx, *filter) } +func (s *LogService) ListUnknownNicks() []*models.UnknownNick { + s.unknownNicksMutex.Lock() + nicks := make([]*models.UnknownNick, 0, len(s.unknownNicks)) + for nick, score := range s.unknownNicks { + nicks = append(nicks, &models.UnknownNick{ + Nick: nick, + Score: score, + }) + } + s.unknownNicksMutex.Unlock() + + sort.Slice(nicks, func(i, j int) bool { + return nicks[i].Score > nicks[j].Score + }) + + return nicks +} + func (s *LogService) Create(ctx context.Context, title, description, channelName, eventName string, open bool) (*models.Log, error) { if channelName == "" { return nil, errors.New("channel name cannot be empty") @@ -280,8 +303,8 @@ func (s *LogService) SplitLog(ctx context.Context, logId string, startPostId str s.changeService.Submit(ctx, models.ChangeModelPost, "add", true, changekeys.Many(newLog, newPosts), newPosts) // Refresh character lists. - _, _ = s.refreshLogCharacters(ctx, *l, nil) - newLog2, err := s.refreshLogCharacters(ctx, *newLog, nil) + _, _ = s.refreshLogCharacters(ctx, *l, nil, false) + newLog2, err := s.refreshLogCharacters(ctx, *newLog, nil, false) if err == nil { newLog = newLog2 } @@ -339,7 +362,7 @@ func (s *LogService) MergeLogs(ctx context.Context, targetID string, sourceID st } // Refresh characters - target2, err := s.refreshLogCharacters(ctx, *target, nil) + target2, err := s.refreshLogCharacters(ctx, *target, nil, false) if err != nil { log.Printf("Failed to update characters in log %s: %s", target.ID, err) } else { @@ -379,7 +402,7 @@ func (s *LogService) AddPost(ctx context.Context, logId string, time time.Time, return nil, err } - l2, err := s.refreshLogCharacters(ctx, *l, nil) + l2, err := s.refreshLogCharacters(ctx, *l, nil, false) if err != nil { log.Printf("Failed to update characters in log %s: %s", l.ID, err) } else { @@ -416,7 +439,7 @@ func (s *LogService) EditPost(ctx context.Context, id string, update models.Post return } - _, err = s.refreshLogCharacters(ctx, *l, nil) + _, err = s.refreshLogCharacters(ctx, *l, nil, false) if err != nil { log.Printf("Failed to update characters in log %s: %s", l.ID, err) } @@ -483,7 +506,7 @@ func (s *LogService) DeletePost(ctx context.Context, id string) (*models.Post, e return } - _, err = s.refreshLogCharacters(ctx, *l, nil) + _, err = s.refreshLogCharacters(ctx, *l, nil, false) if err != nil { log.Printf("Failed to update characters in log %s: %s", l.ID, err) } @@ -638,12 +661,18 @@ func (s *LogService) RefreshAllLogCharacters(ctx context.Context) error { } characterMap := s.makeCharacterMap(characters) + s.unknownNicksMutex.Lock() + for key := range s.unknownNicks { + delete(s.unknownNicks, key) + } + s.unknownNicksMutex.Unlock() + eg := errgroup.Group{} for i := range logs { l := logs[i] eg.Go(func() error { - _, err := s.refreshLogCharacters(ctx, *l, characterMap) + _, err := s.refreshLogCharacters(ctx, *l, characterMap, true) return err }) } @@ -652,16 +681,20 @@ func (s *LogService) RefreshAllLogCharacters(ctx context.Context) error { return err } - log.Printf("Full log character refresh complete; nicks: %d, logs: %d, duration: %s", len(characterMap), len(logs), time.Since(start)) + s.unknownNicksMutex.Lock() + unknownCount := len(s.unknownNicks) + s.unknownNicksMutex.Unlock() + + log.Printf("Full log character refresh complete; nicks: %d, unknowns: %d, logs: %d, duration: %s", len(characterMap), unknownCount, len(logs), time.Since(start)) return nil } func (s *LogService) RefreshLogCharacters(ctx context.Context, log models.Log) (*models.Log, error) { - return s.refreshLogCharacters(ctx, log, nil) + return s.refreshLogCharacters(ctx, log, nil, false) } -func (s *LogService) refreshLogCharacters(ctx context.Context, log models.Log, characterMap map[string]*models.Character) (*models.Log, error) { +func (s *LogService) refreshLogCharacters(ctx context.Context, log models.Log, characterMap map[string]*models.Character, useUnknownNicks bool) (*models.Log, error) { posts, err := s.ListPosts(ctx, &models.PostFilter{LogID: &log.ShortID}) if err != nil { return &log, nil @@ -718,9 +751,19 @@ func (s *LogService) refreshLogCharacters(ctx context.Context, log models.Log, c for key := range added { delete(added, key) } + unknowned := make(map[string]bool) for _, nick := range nicks { character := characterMap[nick] - if character == nil || added[character.ID] { + if character == nil { + if useUnknownNicks && !unknowned[nick] { + unknowned[nick] = true + s.unknownNicksMutex.Lock() + s.unknownNicks[nick]++ + s.unknownNicksMutex.Unlock() + } + + continue + } else if added[character.ID] { continue } added[character.ID] = true diff --git a/services/services.go b/services/services.go index 3a22335..946063c 100644 --- a/services/services.go +++ b/services/services.go @@ -38,6 +38,8 @@ func NewBundle(db database.Database) *Bundle { changeService: bundle.Changes, channelService: bundle.Channels, characterService: bundle.Characters, + + unknownNicks: make(map[string]int, 512), } bundle.Stories = &StoryService{ stories: db.Stories(),