Moved unknown nicks to log service

graph2/resolvers/unknownnicks.go

@@ -5,28 +5,20 @@ import (
 
 	"git.aiterp.net/rpdata/api/graph2/graphcore"
 	"git.aiterp.net/rpdata/api/models"
-	"git.aiterp.net/rpdata/api/models/unknownnicks"
 )
 
 /// Queries
 
 func (r *queryResolver) UnknownNicks(ctx context.Context, filter *graphcore.UnknownNicksFilter) ([]*models.UnknownNick, error) {
-	limit := 100
+	limit := 20
 	if filter != nil {
-		if filter.Limit > 0 {
-			limit = filter.Limit
-		}
+		limit = filter.Limit
 	}
 
-	nicks, err := unknownnicks.List(limit)
-	if err != nil {
-		return nil, err
+	nicks := r.s.Logs.ListUnknownNicks()
+	if limit > 0 && len(nicks) > limit {
+		nicks = nicks[:limit]
 	}
 
-	nicks2 := make([]*models.UnknownNick, len(nicks))
-	for i := range nicks {
-		nicks2[i] = &nicks[i]
-	}
-
-	return nicks2, nil
+	return nicks, nil
 }

models/unknownnicks/db.go

@@ -1,37 +0,0 @@
-package unknownnicks
-
-import (
-	"log"
-
-	"git.aiterp.net/rpdata/api/internal/store"
-	"git.aiterp.net/rpdata/api/models"
-	"github.com/globalsign/mgo"
-)
-
-var collection *mgo.Collection
-
-func list(query interface{}, limit int) ([]models.UnknownNick, error) {
-	size := 64
-	if limit > 0 {
-		size = limit
-	}
-	posts := make([]models.UnknownNick, 0, size)
-
-	err := collection.Find(query).Limit(limit).Sort("-score").All(&posts)
-	if err != nil {
-		return nil, err
-	}
-
-	return posts, nil
-}
-
-func init() {
-	store.HandleInit(func(db *mgo.Database) {
-		collection = db.C("logbot3.unkownnicks")
-
-		err := collection.EnsureIndexKey("weight")
-		if err != nil {
-			log.Fatalln("init logbot3.unkownnicks:", err)
-		}
-	})
-}

models/unknownnicks/list.go

@@ -1,10 +0,0 @@
-package unknownnicks
-
-import (
-	"git.aiterp.net/rpdata/api/models"
-)
-
-// List lists the top N unknown nicks.
-func List(limit int) ([]models.UnknownNick, error) {
-	return list(nil, limit)
-}

models/unknownnicks/remove.go

@@ -1 +0,0 @@
-package unknownnicks

models/unknownnicks/update.go

@@ -1,22 +0,0 @@
-package unknownnicks
-
-import (
-	"github.com/globalsign/mgo/bson"
-)
-
-// Update updates the map, replacing it with the scores provided here.
-func Update(updateMap map[string]int) error {
-	_, err := collection.RemoveAll(bson.M{})
-	if err != nil {
-		return err
-	}
-
-	for nick, score := range updateMap {
-		_, err := collection.UpsertId(nick, bson.M{"$set": bson.M{"score": score}})
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

services/logs.go

@@ -11,7 +11,9 @@ import (
 	"git.aiterp.net/rpdata/api/services/parsers"
 	"golang.org/x/sync/errgroup"
 	"log"
+	"sort"
 	"strings"
+	"sync"
 	"time"
 )
 
@@ -21,6 +23,9 @@ type LogService struct {
 	changeService    *ChangeService
 	channelService   *ChannelService
 	characterService *CharacterService
+
+	unknownNicks      map[string]int
+	unknownNicksMutex sync.Mutex
 }
 
 func (s *LogService) Find(ctx context.Context, id string) (*models.Log, error) {
@@ -56,6 +61,24 @@ func (s *LogService) ListPosts(ctx context.Context, filter *models.PostFilter) (
 	return s.posts.List(ctx, *filter)
 }
 
+func (s *LogService) ListUnknownNicks() []*models.UnknownNick {
+	s.unknownNicksMutex.Lock()
+	nicks := make([]*models.UnknownNick, 0, len(s.unknownNicks))
+	for nick, score := range s.unknownNicks {
+		nicks = append(nicks, &models.UnknownNick{
+			Nick:  nick,
+			Score: score,
+		})
+	}
+	s.unknownNicksMutex.Unlock()
+
+	sort.Slice(nicks, func(i, j int) bool {
+		return nicks[i].Score > nicks[j].Score
+	})
+
+	return nicks
+}
+
 func (s *LogService) Create(ctx context.Context, title, description, channelName, eventName string, open bool) (*models.Log, error) {
 	if channelName == "" {
 		return nil, errors.New("channel name cannot be empty")
@@ -280,8 +303,8 @@ func (s *LogService) SplitLog(ctx context.Context, logId string, startPostId str
 	s.changeService.Submit(ctx, models.ChangeModelPost, "add", true, changekeys.Many(newLog, newPosts), newPosts)
 
 	// Refresh character lists.
-	_, _ = s.refreshLogCharacters(ctx, *l, nil)
-	newLog2, err := s.refreshLogCharacters(ctx, *newLog, nil)
+	_, _ = s.refreshLogCharacters(ctx, *l, nil, false)
+	newLog2, err := s.refreshLogCharacters(ctx, *newLog, nil, false)
 	if err == nil {
 		newLog = newLog2
 	}
@@ -339,7 +362,7 @@ func (s *LogService) MergeLogs(ctx context.Context, targetID string, sourceID st
 	}
 
 	// Refresh characters
-	target2, err := s.refreshLogCharacters(ctx, *target, nil)
+	target2, err := s.refreshLogCharacters(ctx, *target, nil, false)
 	if err != nil {
 		log.Printf("Failed to update characters in log %s: %s", target.ID, err)
 	} else {
@@ -379,7 +402,7 @@ func (s *LogService) AddPost(ctx context.Context, logId string, time time.Time,
 		return nil, err
 	}
 
-	l2, err := s.refreshLogCharacters(ctx, *l, nil)
+	l2, err := s.refreshLogCharacters(ctx, *l, nil, false)
 	if err != nil {
 		log.Printf("Failed to update characters in log %s: %s", l.ID, err)
 	} else {
@@ -416,7 +439,7 @@ func (s *LogService) EditPost(ctx context.Context, id string, update models.Post
 			return
 		}
 
-		_, err = s.refreshLogCharacters(ctx, *l, nil)
+		_, err = s.refreshLogCharacters(ctx, *l, nil, false)
 		if err != nil {
 			log.Printf("Failed to update characters in log %s: %s", l.ID, err)
 		}
@@ -483,7 +506,7 @@ func (s *LogService) DeletePost(ctx context.Context, id string) (*models.Post, e
 			return
 		}
 
-		_, err = s.refreshLogCharacters(ctx, *l, nil)
+		_, err = s.refreshLogCharacters(ctx, *l, nil, false)
 		if err != nil {
 			log.Printf("Failed to update characters in log %s: %s", l.ID, err)
 		}
@@ -638,12 +661,18 @@ func (s *LogService) RefreshAllLogCharacters(ctx context.Context) error {
 	}
 	characterMap := s.makeCharacterMap(characters)
 
+	s.unknownNicksMutex.Lock()
+	for key := range s.unknownNicks {
+		delete(s.unknownNicks, key)
+	}
+	s.unknownNicksMutex.Unlock()
+
 	eg := errgroup.Group{}
 	for i := range logs {
 		l := logs[i]
 
 		eg.Go(func() error {
-			_, err := s.refreshLogCharacters(ctx, *l, characterMap)
+			_, err := s.refreshLogCharacters(ctx, *l, characterMap, true)
 			return err
 		})
 	}
@@ -652,16 +681,20 @@ func (s *LogService) RefreshAllLogCharacters(ctx context.Context) error {
 		return err
 	}
 
-	log.Printf("Full log character refresh complete; nicks: %d, logs: %d, duration: %s", len(characterMap), len(logs), time.Since(start))
+	s.unknownNicksMutex.Lock()
+	unknownCount := len(s.unknownNicks)
+	s.unknownNicksMutex.Unlock()
+
+	log.Printf("Full log character refresh complete; nicks: %d, unknowns: %d, logs: %d, duration: %s", len(characterMap), unknownCount, len(logs), time.Since(start))
 
 	return nil
 }
 
 func (s *LogService) RefreshLogCharacters(ctx context.Context, log models.Log) (*models.Log, error) {
-	return s.refreshLogCharacters(ctx, log, nil)
+	return s.refreshLogCharacters(ctx, log, nil, false)
 }
 
-func (s *LogService) refreshLogCharacters(ctx context.Context, log models.Log, characterMap map[string]*models.Character) (*models.Log, error) {
+func (s *LogService) refreshLogCharacters(ctx context.Context, log models.Log, characterMap map[string]*models.Character, useUnknownNicks bool) (*models.Log, error) {
 	posts, err := s.ListPosts(ctx, &models.PostFilter{LogID: &log.ShortID})
 	if err != nil {
 		return &log, nil
@@ -718,9 +751,19 @@ func (s *LogService) refreshLogCharacters(ctx context.Context, log models.Log, c
 	for key := range added {
 		delete(added, key)
 	}
+	unknowned := make(map[string]bool)
 	for _, nick := range nicks {
 		character := characterMap[nick]
-		if character == nil || added[character.ID] {
+		if character == nil {
+			if useUnknownNicks && !unknowned[nick] {
+				unknowned[nick] = true
+				s.unknownNicksMutex.Lock()
+				s.unknownNicks[nick]++
+				s.unknownNicksMutex.Unlock()
+			}
+
+			continue
+		} else if added[character.ID] {
 			continue
 		}
 		added[character.ID] = true

services/services.go

@@ -38,6 +38,8 @@ func NewBundle(db database.Database) *Bundle {
 		changeService:    bundle.Changes,
 		channelService:   bundle.Channels,
 		characterService: bundle.Characters,
+
+		unknownNicks: make(map[string]int, 512),
 	}
 	bundle.Stories = &StoryService{
 		stories:          db.Stories(),