Added log character refresh function to log service.

cmd/rpdata-server/main.go

@@ -16,7 +16,6 @@ import (
 	"git.aiterp.net/rpdata/api/internal/loader"
 	"git.aiterp.net/rpdata/api/internal/store"
 	"git.aiterp.net/rpdata/api/models"
-	"git.aiterp.net/rpdata/api/models/logs"
 	"git.aiterp.net/rpdata/api/services"
 	"github.com/99designs/gqlgen/handler"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -43,7 +42,7 @@ func main() {
 	http.Handle("/metrics", promhttp.Handler())
 
 	go func() {
-		err := logs.RunFullUpdate()
+		err := serviceBundle.Logs.RefreshAllLogCharacters(context.Background())
 		if err != nil {
 			log.Println(err)
 		}

database/mongodb/logs.go

@@ -172,6 +172,10 @@ func (r *logRepository) Update(ctx context.Context, log models.Log, update model
 		updateBson["event"] = *update.EventName
 		log.EventName = *update.EventName
 	}
+	if update.CharacterIDs != nil {
+		updateBson["characterIds"] = update.CharacterIDs
+		log.CharacterIDs = update.CharacterIDs
+	}
 
 	err := r.logs.UpdateId(log.ID, bson.M{"$set": updateBson})
 	if err != nil {

go.mod

@@ -31,6 +31,7 @@ require (
 	go.mongodb.org/mongo-driver v1.0.3
 	golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 // indirect
 	golang.org/x/net v0.0.0-20190514140710-3ec191127204 // indirect
+	golang.org/x/sync v0.0.0-20190423024810-112230192c58
 	golang.org/x/text v0.3.0 // indirect
 	google.golang.org/appengine v1.1.0 // indirect
 	gopkg.in/ini.v1 v1.42.0 // indirect

go.sum

@@ -124,6 +124,7 @@ golang.org/x/net v0.0.0-20180416171110-a35a21de978d h1:O2P57H5Cc+d+DJos+iweraI9r
 golang.org/x/net v0.0.0-20180416171110-a35a21de978d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5 h1:mzjBh+S5frKOsOBobWIMAbXavqjmgO17k/2puhcFR94=

models/log.go

@@ -32,8 +32,9 @@ type LogFilter struct {
 }
 
 type LogUpdate struct {
-	Title       *string
+	Title        *string
-	EventName   *string
+	EventName    *string
-	Description *string
+	Description  *string
-	Open        *bool
+	Open         *bool
+	CharacterIDs []string
 }

services/logs.go

@@ -9,14 +9,18 @@ import (
 	"git.aiterp.net/rpdata/api/models/channels"
 	"git.aiterp.net/rpdata/api/repositories"
 	"git.aiterp.net/rpdata/api/services/parsers"
+	"golang.org/x/sync/errgroup"
+	"log"
+	"strings"
 	"time"
 )
 
 type LogService struct {
-	logs           repositories.LogRepository
+	logs             repositories.LogRepository
-	posts          repositories.PostRepository
+	posts            repositories.PostRepository
-	changeService  *ChangeService
+	changeService    *ChangeService
-	channelService *ChannelService
+	channelService   *ChannelService
+	characterService *CharacterService
 }
 
 func (s *LogService) Find(ctx context.Context, id string) (*models.Log, error) {
@@ -339,3 +343,123 @@ func (s *LogService) Delete(ctx context.Context, id string) (*models.Log, error)
 
 	return log, nil
 }
+
+func (s *LogService) RefreshAllLogCharacters(ctx context.Context) error {
+	start := time.Now()
+
+	// Get all logs
+	logs, err := s.logs.List(ctx, models.LogFilter{})
+	if err != nil {
+		return err
+	}
+
+	// Check all characters now instead of later.
+	characters, err := s.characterService.List(ctx, models.CharacterFilter{})
+	if err != nil {
+		return err
+	}
+	characterMap := s.makeCharacterMap(characters)
+
+	eg := errgroup.Group{}
+	for i := range logs {
+		l := logs[i]
+
+		eg.Go(func() error {
+			return s.refreshLogCharacters(ctx, *l, characterMap)
+		})
+	}
+	err = eg.Wait()
+	if err != nil {
+		return err
+	}
+
+	log.Printf("Full log character refresh complete; nicks: %d, logs: %d, duration: %s", len(characterMap), len(logs), time.Since(start))
+
+	return nil
+}
+
+func (s *LogService) RefreshLogCharacters(ctx context.Context, log models.Log) error {
+	return s.refreshLogCharacters(ctx, log, nil)
+}
+
+func (s *LogService) refreshLogCharacters(ctx context.Context, log models.Log, characterMap map[string]*models.Character) error {
+	posts, err := s.ListPosts(ctx, &models.PostFilter{LogID: &log.ShortID})
+	if err != nil {
+		return nil
+	}
+
+	counts := make(map[string]int)
+	added := make(map[string]bool)
+	removed := make(map[string]bool)
+	for _, post := range posts {
+		if post.Kind == "text" || post.Kind == "action" {
+			if strings.HasPrefix(post.Text, "(") || strings.Contains(post.Nick, "(") || strings.Contains(post.Nick, "[E]") || strings.HasSuffix(post.Nick, "|") {
+				continue
+			}
+
+			// Clean up the nick (remove possessive suffix, comma, formatting stuff)
+			if strings.HasSuffix(post.Nick, "'s") || strings.HasSuffix(post.Nick, "`s") {
+				post.Nick = post.Nick[:len(post.Nick)-2]
+			} else if strings.HasSuffix(post.Nick, "'") || strings.HasSuffix(post.Nick, "`") || strings.HasSuffix(post.Nick, ",") || strings.HasSuffix(post.Nick, "\x0f") {
+				post.Nick = post.Nick[:len(post.Nick)-1]
+			}
+
+			added[post.Nick] = true
+			counts[post.Nick]++
+		}
+		if post.Kind == "chars" {
+			tokens := strings.Fields(post.Text)
+			for _, token := range tokens {
+				if strings.HasPrefix(token, "-") {
+					removed[token[1:]] = true
+				} else {
+					added[strings.Replace(token, "+", "", 1)] = true
+				}
+			}
+		}
+	}
+
+	nicks := make([]string, 0, len(added))
+	for nick := range added {
+		if added[nick] && !removed[nick] {
+			nicks = append(nicks, nick)
+		}
+	}
+
+	if characterMap == nil {
+		characters, err := s.characterService.List(ctx, models.CharacterFilter{Nicks: nicks})
+		if err != nil {
+			return err
+		}
+
+		characterMap = s.makeCharacterMap(characters)
+	}
+
+	log.CharacterIDs = log.CharacterIDs[:0]
+	for key := range added {
+		delete(added, key)
+	}
+	for _, nick := range nicks {
+		character := characterMap[nick]
+		if character == nil || added[character.ID] {
+			continue
+		}
+		added[character.ID] = true
+
+		log.CharacterIDs = append(log.CharacterIDs, character.ID)
+	}
+
+	_, err = s.logs.Update(ctx, log, models.LogUpdate{CharacterIDs: log.CharacterIDs})
+	return err
+}
+
+func (s *LogService) makeCharacterMap(characters []*models.Character) map[string]*models.Character {
+	characterMap := make(map[string]*models.Character, len(characters)*3)
+	for _, character := range characters {
+		for _, nick := range character.Nicks {
+			characterMap[nick] = character
+		}
+	}
+
+	return characterMap
+}

services/services.go

@@ -32,9 +32,11 @@ func NewBundle(db database.Database) *Bundle {
 		changeService: bundle.Changes,
 	}
 	bundle.Logs = &LogService{
-		logs:          db.Logs(),
+		logs:             db.Logs(),
-		posts:         db.Posts(),
+		posts:            db.Posts(),
-		changeService: bundle.Changes,
+		changeService:    bundle.Changes,
+		channelService:   bundle.Channels,
+		characterService: bundle.Characters,
 	}
 
 	return bundle