From c0687d76ad1de7870dd1569e28d541df0083986d Mon Sep 17 00:00:00 2001 From: Gisle Aune Date: Wed, 7 Aug 2019 12:26:59 +0200 Subject: [PATCH] Added log character refresh function to log service. --- cmd/rpdata-server/main.go | 3 +- database/mongodb/logs.go | 4 ++ go.mod | 1 + go.sum | 1 + models/log.go | 9 +-- services/logs.go | 132 ++++++++++++++++++++++++++++++++++++-- services/services.go | 8 ++- 7 files changed, 145 insertions(+), 13 deletions(-) diff --git a/cmd/rpdata-server/main.go b/cmd/rpdata-server/main.go index ef84f6f..480686c 100644 --- a/cmd/rpdata-server/main.go +++ b/cmd/rpdata-server/main.go @@ -16,7 +16,6 @@ import ( "git.aiterp.net/rpdata/api/internal/loader" "git.aiterp.net/rpdata/api/internal/store" "git.aiterp.net/rpdata/api/models" - "git.aiterp.net/rpdata/api/models/logs" "git.aiterp.net/rpdata/api/services" "github.com/99designs/gqlgen/handler" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -43,7 +42,7 @@ func main() { http.Handle("/metrics", promhttp.Handler()) go func() { - err := logs.RunFullUpdate() + err := serviceBundle.Logs.RefreshAllLogCharacters(context.Background()) if err != nil { log.Println(err) } diff --git a/database/mongodb/logs.go b/database/mongodb/logs.go index cefd975..2e87a1b 100644 --- a/database/mongodb/logs.go +++ b/database/mongodb/logs.go @@ -172,6 +172,10 @@ func (r *logRepository) Update(ctx context.Context, log models.Log, update model updateBson["event"] = *update.EventName log.EventName = *update.EventName } + if update.CharacterIDs != nil { + updateBson["characterIds"] = update.CharacterIDs + log.CharacterIDs = update.CharacterIDs + } err := r.logs.UpdateId(log.ID, bson.M{"$set": updateBson}) if err != nil { diff --git a/go.mod b/go.mod index e7a9205..a6d2f80 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( go.mongodb.org/mongo-driver v1.0.3 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 // indirect golang.org/x/net v0.0.0-20190514140710-3ec191127204 // indirect + golang.org/x/sync v0.0.0-20190423024810-112230192c58 golang.org/x/text v0.3.0 // indirect google.golang.org/appengine v1.1.0 // indirect gopkg.in/ini.v1 v1.42.0 // indirect diff --git a/go.sum b/go.sum index b67cadb..0383972 100644 --- a/go.sum +++ b/go.sum @@ -124,6 +124,7 @@ golang.org/x/net v0.0.0-20180416171110-a35a21de978d h1:O2P57H5Cc+d+DJos+iweraI9r golang.org/x/net v0.0.0-20180416171110-a35a21de978d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5 h1:mzjBh+S5frKOsOBobWIMAbXavqjmgO17k/2puhcFR94= diff --git a/models/log.go b/models/log.go index d4d82d2..91ffb2d 100644 --- a/models/log.go +++ b/models/log.go @@ -32,8 +32,9 @@ type LogFilter struct { } type LogUpdate struct { - Title *string - EventName *string - Description *string - Open *bool + Title *string + EventName *string + Description *string + Open *bool + CharacterIDs []string } diff --git a/services/logs.go b/services/logs.go index 571aa5f..c0a7882 100644 --- a/services/logs.go +++ b/services/logs.go @@ -9,14 +9,18 @@ import ( "git.aiterp.net/rpdata/api/models/channels" "git.aiterp.net/rpdata/api/repositories" "git.aiterp.net/rpdata/api/services/parsers" + "golang.org/x/sync/errgroup" + "log" + "strings" "time" ) type LogService struct { - logs repositories.LogRepository - posts repositories.PostRepository - changeService *ChangeService - channelService *ChannelService + logs repositories.LogRepository + posts repositories.PostRepository + changeService *ChangeService + channelService *ChannelService + characterService *CharacterService } func (s *LogService) Find(ctx context.Context, id string) (*models.Log, error) { @@ -339,3 +343,123 @@ func (s *LogService) Delete(ctx context.Context, id string) (*models.Log, error) return log, nil } + +func (s *LogService) RefreshAllLogCharacters(ctx context.Context) error { + start := time.Now() + + // Get all logs + logs, err := s.logs.List(ctx, models.LogFilter{}) + if err != nil { + return err + } + + // Check all characters now instead of later. + characters, err := s.characterService.List(ctx, models.CharacterFilter{}) + if err != nil { + return err + } + characterMap := s.makeCharacterMap(characters) + + eg := errgroup.Group{} + for i := range logs { + l := logs[i] + + eg.Go(func() error { + return s.refreshLogCharacters(ctx, *l, characterMap) + }) + } + err = eg.Wait() + if err != nil { + return err + } + + log.Printf("Full log character refresh complete; nicks: %d, logs: %d, duration: %s", len(characterMap), len(logs), time.Since(start)) + + return nil +} + +func (s *LogService) RefreshLogCharacters(ctx context.Context, log models.Log) error { + return s.refreshLogCharacters(ctx, log, nil) +} + +func (s *LogService) refreshLogCharacters(ctx context.Context, log models.Log, characterMap map[string]*models.Character) error { + posts, err := s.ListPosts(ctx, &models.PostFilter{LogID: &log.ShortID}) + if err != nil { + return nil + } + + counts := make(map[string]int) + added := make(map[string]bool) + removed := make(map[string]bool) + for _, post := range posts { + if post.Kind == "text" || post.Kind == "action" { + if strings.HasPrefix(post.Text, "(") || strings.Contains(post.Nick, "(") || strings.Contains(post.Nick, "[E]") || strings.HasSuffix(post.Nick, "|") { + continue + } + + // Clean up the nick (remove possessive suffix, comma, formatting stuff) + if strings.HasSuffix(post.Nick, "'s") || strings.HasSuffix(post.Nick, "`s") { + post.Nick = post.Nick[:len(post.Nick)-2] + } else if strings.HasSuffix(post.Nick, "'") || strings.HasSuffix(post.Nick, "`") || strings.HasSuffix(post.Nick, ",") || strings.HasSuffix(post.Nick, "\x0f") { + post.Nick = post.Nick[:len(post.Nick)-1] + } + + added[post.Nick] = true + counts[post.Nick]++ + } + if post.Kind == "chars" { + tokens := strings.Fields(post.Text) + for _, token := range tokens { + if strings.HasPrefix(token, "-") { + removed[token[1:]] = true + } else { + added[strings.Replace(token, "+", "", 1)] = true + } + } + } + } + + nicks := make([]string, 0, len(added)) + for nick := range added { + if added[nick] && !removed[nick] { + nicks = append(nicks, nick) + } + } + + if characterMap == nil { + characters, err := s.characterService.List(ctx, models.CharacterFilter{Nicks: nicks}) + if err != nil { + return err + } + + characterMap = s.makeCharacterMap(characters) + } + + log.CharacterIDs = log.CharacterIDs[:0] + for key := range added { + delete(added, key) + } + for _, nick := range nicks { + character := characterMap[nick] + if character == nil || added[character.ID] { + continue + } + added[character.ID] = true + + log.CharacterIDs = append(log.CharacterIDs, character.ID) + } + + _, err = s.logs.Update(ctx, log, models.LogUpdate{CharacterIDs: log.CharacterIDs}) + return err +} + +func (s *LogService) makeCharacterMap(characters []*models.Character) map[string]*models.Character { + characterMap := make(map[string]*models.Character, len(characters)*3) + for _, character := range characters { + for _, nick := range character.Nicks { + characterMap[nick] = character + } + } + + return characterMap +} diff --git a/services/services.go b/services/services.go index 1641d4c..b03af15 100644 --- a/services/services.go +++ b/services/services.go @@ -32,9 +32,11 @@ func NewBundle(db database.Database) *Bundle { changeService: bundle.Changes, } bundle.Logs = &LogService{ - logs: db.Logs(), - posts: db.Posts(), - changeService: bundle.Changes, + logs: db.Logs(), + posts: db.Posts(), + changeService: bundle.Changes, + channelService: bundle.Channels, + characterService: bundle.Characters, } return bundle