Moved unknown nicks to log service

services/logs.go

@@ -11,7 +11,9 @@ import (
 	"git.aiterp.net/rpdata/api/services/parsers"
 	"golang.org/x/sync/errgroup"
 	"log"
+	"sort"
 	"strings"
+	"sync"
 	"time"
 )
 
@@ -21,6 +23,9 @@ type LogService struct {
 	changeService    *ChangeService
 	channelService   *ChannelService
 	characterService *CharacterService
+
+	unknownNicks      map[string]int
+	unknownNicksMutex sync.Mutex
 }
 
 func (s *LogService) Find(ctx context.Context, id string) (*models.Log, error) {
@@ -56,6 +61,24 @@ func (s *LogService) ListPosts(ctx context.Context, filter *models.PostFilter) (
 	return s.posts.List(ctx, *filter)
 }
 
+func (s *LogService) ListUnknownNicks() []*models.UnknownNick {
+	s.unknownNicksMutex.Lock()
+	nicks := make([]*models.UnknownNick, 0, len(s.unknownNicks))
+	for nick, score := range s.unknownNicks {
+		nicks = append(nicks, &models.UnknownNick{
+			Nick:  nick,
+			Score: score,
+		})
+	}
+	s.unknownNicksMutex.Unlock()
+
+	sort.Slice(nicks, func(i, j int) bool {
+		return nicks[i].Score > nicks[j].Score
+	})
+
+	return nicks
+}
+
 func (s *LogService) Create(ctx context.Context, title, description, channelName, eventName string, open bool) (*models.Log, error) {
 	if channelName == "" {
 		return nil, errors.New("channel name cannot be empty")
@@ -280,8 +303,8 @@ func (s *LogService) SplitLog(ctx context.Context, logId string, startPostId str
 	s.changeService.Submit(ctx, models.ChangeModelPost, "add", true, changekeys.Many(newLog, newPosts), newPosts)
 
 	// Refresh character lists.
-	_, _ = s.refreshLogCharacters(ctx, *l, nil)
-	newLog2, err := s.refreshLogCharacters(ctx, *newLog, nil)
+	_, _ = s.refreshLogCharacters(ctx, *l, nil, false)
+	newLog2, err := s.refreshLogCharacters(ctx, *newLog, nil, false)
 	if err == nil {
 		newLog = newLog2
 	}
@@ -339,7 +362,7 @@ func (s *LogService) MergeLogs(ctx context.Context, targetID string, sourceID st
 	}
 
 	// Refresh characters
-	target2, err := s.refreshLogCharacters(ctx, *target, nil)
+	target2, err := s.refreshLogCharacters(ctx, *target, nil, false)
 	if err != nil {
 		log.Printf("Failed to update characters in log %s: %s", target.ID, err)
 	} else {
@@ -379,7 +402,7 @@ func (s *LogService) AddPost(ctx context.Context, logId string, time time.Time,
 		return nil, err
 	}
 
-	l2, err := s.refreshLogCharacters(ctx, *l, nil)
+	l2, err := s.refreshLogCharacters(ctx, *l, nil, false)
 	if err != nil {
 		log.Printf("Failed to update characters in log %s: %s", l.ID, err)
 	} else {
@@ -416,7 +439,7 @@ func (s *LogService) EditPost(ctx context.Context, id string, update models.Post
 			return
 		}
 
-		_, err = s.refreshLogCharacters(ctx, *l, nil)
+		_, err = s.refreshLogCharacters(ctx, *l, nil, false)
 		if err != nil {
 			log.Printf("Failed to update characters in log %s: %s", l.ID, err)
 		}
@@ -483,7 +506,7 @@ func (s *LogService) DeletePost(ctx context.Context, id string) (*models.Post, e
 			return
 		}
 
-		_, err = s.refreshLogCharacters(ctx, *l, nil)
+		_, err = s.refreshLogCharacters(ctx, *l, nil, false)
 		if err != nil {
 			log.Printf("Failed to update characters in log %s: %s", l.ID, err)
 		}
@@ -638,12 +661,18 @@ func (s *LogService) RefreshAllLogCharacters(ctx context.Context) error {
 	}
 	characterMap := s.makeCharacterMap(characters)
 
+	s.unknownNicksMutex.Lock()
+	for key := range s.unknownNicks {
+		delete(s.unknownNicks, key)
+	}
+	s.unknownNicksMutex.Unlock()
+
 	eg := errgroup.Group{}
 	for i := range logs {
 		l := logs[i]
 
 		eg.Go(func() error {
-			_, err := s.refreshLogCharacters(ctx, *l, characterMap)
+			_, err := s.refreshLogCharacters(ctx, *l, characterMap, true)
 			return err
 		})
 	}
@@ -652,16 +681,20 @@ func (s *LogService) RefreshAllLogCharacters(ctx context.Context) error {
 		return err
 	}
 
-	log.Printf("Full log character refresh complete; nicks: %d, logs: %d, duration: %s", len(characterMap), len(logs), time.Since(start))
+	s.unknownNicksMutex.Lock()
+	unknownCount := len(s.unknownNicks)
+	s.unknownNicksMutex.Unlock()
+
+	log.Printf("Full log character refresh complete; nicks: %d, unknowns: %d, logs: %d, duration: %s", len(characterMap), unknownCount, len(logs), time.Since(start))
 
 	return nil
 }
 
 func (s *LogService) RefreshLogCharacters(ctx context.Context, log models.Log) (*models.Log, error) {
-	return s.refreshLogCharacters(ctx, log, nil)
+	return s.refreshLogCharacters(ctx, log, nil, false)
 }
 
-func (s *LogService) refreshLogCharacters(ctx context.Context, log models.Log, characterMap map[string]*models.Character) (*models.Log, error) {
+func (s *LogService) refreshLogCharacters(ctx context.Context, log models.Log, characterMap map[string]*models.Character, useUnknownNicks bool) (*models.Log, error) {
 	posts, err := s.ListPosts(ctx, &models.PostFilter{LogID: &log.ShortID})
 	if err != nil {
 		return &log, nil
@@ -718,9 +751,19 @@ func (s *LogService) refreshLogCharacters(ctx context.Context, log models.Log, c
 	for key := range added {
 		delete(added, key)
 	}
+	unknowned := make(map[string]bool)
 	for _, nick := range nicks {
 		character := characterMap[nick]
-		if character == nil || added[character.ID] {
+		if character == nil {
+			if useUnknownNicks && !unknowned[nick] {
+				unknowned[nick] = true
+				s.unknownNicksMutex.Lock()
+				s.unknownNicks[nick]++
+				s.unknownNicksMutex.Unlock()
+			}
+
+			continue
+		} else if added[character.ID] {
 			continue
 		}
 		added[character.ID] = true

services/services.go

@@ -38,6 +38,8 @@ func NewBundle(db database.Database) *Bundle {
 		changeService:    bundle.Changes,
 		channelService:   bundle.Channels,
 		characterService: bundle.Characters,
+
+		unknownNicks: make(map[string]int, 512),
 	}
 	bundle.Stories = &StoryService{
 		stories:          db.Stories(),