GraphQL API and utilities for the rpdata project
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 

944 lines
23 KiB

package services
import (
"context"
"errors"
"git.aiterp.net/rpdata/api/models"
"git.aiterp.net/rpdata/api/models/changekeys"
"git.aiterp.net/rpdata/api/repositories"
"git.aiterp.net/rpdata/api/services/parsers"
"golang.org/x/sync/errgroup"
"log"
"sort"
"strings"
"sync"
"time"
)
type LogService struct {
logs repositories.LogRepository
posts repositories.PostRepository
changeService *ChangeService
channelService *ChannelService
characterService *CharacterService
authService *AuthService
unknownNicks map[string]int
unknownNicksMutex sync.Mutex
}
func (s *LogService) Find(ctx context.Context, id string) (*models.Log, error) {
return s.logs.Find(ctx, id)
}
func (s *LogService) FindPosts(ctx context.Context, id string) (*models.Post, error) {
return s.posts.Find(ctx, id)
}
func (s *LogService) List(ctx context.Context, filter *models.LogFilter) ([]*models.Log, error) {
if filter == nil {
filter = &models.LogFilter{}
}
return s.logs.List(ctx, *filter)
}
func (s *LogService) ListPosts(ctx context.Context, filter *models.PostFilter) ([]*models.Post, error) {
// Some sanity checks to avoid querying an insame amount of posts.
if filter == nil {
filter = &models.PostFilter{Limit: 100}
} else {
if (filter.Limit <= 0 || filter.Limit > 512) && (filter.LogID == nil && len(filter.IDs) == 0) {
return nil, errors.New("a limit of 0 (no limit) or >512 without a logId or a set of IDs is not allowed")
}
if len(filter.IDs) > 100 {
return nil, errors.New("you may not query for more than 100 ids, split your query")
}
}
return s.posts.List(ctx, *filter)
}
func (s *LogService) ListUnknownNicks() []*models.UnknownNick {
s.unknownNicksMutex.Lock()
nicks := make([]*models.UnknownNick, 0, len(s.unknownNicks))
for nick, score := range s.unknownNicks {
nicks = append(nicks, &models.UnknownNick{
Nick: nick,
Score: score,
})
}
s.unknownNicksMutex.Unlock()
sort.Slice(nicks, func(i, j int) bool {
return nicks[i].Score > nicks[j].Score
})
return nicks
}
func (s *LogService) Create(ctx context.Context, title, description, channelName, eventName string, open bool) (*models.Log, error) {
if channelName == "" {
return nil, errors.New("channel name cannot be empty")
}
log := &models.Log{
Title: title,
Description: description,
ChannelName: channelName,
EventName: eventName,
Date: time.Now(),
Open: open,
}
if err := s.authService.CheckPermission(ctx, "add", log); err != nil {
return nil, err
}
_, err := s.channelService.Ensure(ctx, channelName)
if err != nil {
return nil, err
}
log, err = s.logs.Insert(ctx, *log)
if err != nil {
return nil, err
}
s.changeService.Submit(ctx, models.ChangeModelLog, "add", true, changekeys.Listed(log), log)
return log, nil
}
// Import creates new logs from common formats.
func (s *LogService) Import(ctx context.Context, importer models.LogImporter, date time.Time, tz *time.Location, channelName string, sessionThreshold time.Duration, data string) ([]*models.Log, error) {
if err := s.authService.CheckPermission(ctx, "add", &models.Log{}); err != nil {
return nil, err
}
results := make([]*models.Log, 0, 8)
_, err := s.channelService.Ensure(ctx, channelName)
if err != nil {
return nil, err
}
eventName := ""
if channel, err := s.channelService.Find(ctx, channelName); err == nil {
eventName = channel.EventName
}
date = date.In(tz)
switch importer {
case models.LogImporterMircLike:
{
if date.IsZero() {
return nil, errors.New("date is not optional for mirc-like logs")
}
parsed, err := parsers.MircLog(data, date, true)
if err != nil {
return nil, err
}
parsed.Log.EventName = eventName
parsed.Log.ChannelName = channelName
newLog, err := s.logs.Insert(ctx, parsed.Log)
if err != nil {
return nil, err
}
for _, post := range parsed.Posts {
post.LogID = newLog.ShortID
}
posts, err := s.posts.InsertMany(ctx, parsed.Posts...)
if err != nil {
_ = s.logs.Delete(ctx, *newLog)
return nil, err
}
s.changeService.Submit(ctx, models.ChangeModelLog, "add", true, changekeys.Listed(newLog), newLog)
s.changeService.Submit(ctx, models.ChangeModelPost, "add", true, changekeys.Listed(newLog, posts), newLog, posts)
refreshedLog, err := s.refreshLogCharacters(ctx, *newLog, nil, false)
if err != nil {
log.Printf("Failed to update characters in newLog %s: %s", newLog.ID, err)
} else {
newLog = refreshedLog
}
results = append(results, newLog)
}
case models.LogImporterForumLog:
{
parseResults, err := parsers.ForumLog(data, tz)
if err != nil {
return nil, err
}
for _, parsed := range parseResults {
parsed.Log.EventName = eventName
parsed.Log.ChannelName = channelName
newLog, err := s.logs.Insert(ctx, parsed.Log)
if err != nil {
return nil, err
}
for _, post := range parsed.Posts {
post.LogID = newLog.ShortID
}
posts, err := s.posts.InsertMany(ctx, parsed.Posts...)
if err != nil {
_ = s.logs.Delete(ctx, *newLog)
return nil, err
}
s.changeService.Submit(ctx, models.ChangeModelLog, "add", true, changekeys.Listed(newLog), newLog)
s.changeService.Submit(ctx, models.ChangeModelPost, "add", true, changekeys.Listed(newLog, posts), newLog, posts)
results = append(results, newLog)
}
}
case models.LogImporterIrcCloud:
{
parseResults, err := parsers.IRCCloudLogs(data, tz, sessionThreshold)
if err != nil {
return nil, err
}
for _, parsed := range parseResults {
parsed.Log.EventName = eventName
parsed.Log.ChannelName = channelName
newLog, err := s.logs.Insert(ctx, parsed.Log)
if err != nil {
return nil, err
}
for _, post := range parsed.Posts {
post.LogID = newLog.ShortID
}
posts, err := s.posts.InsertMany(ctx, parsed.Posts...)
if err != nil {
_ = s.logs.Delete(ctx, *newLog)
return results, err
}
refreshedLog, err := s.refreshLogCharacters(ctx, *newLog, nil, false)
if err != nil {
log.Printf("Failed to update characters in newLog %s: %s", newLog.ID, err)
} else {
newLog = refreshedLog
}
s.changeService.Submit(ctx, models.ChangeModelLog, "add", true, changekeys.Listed(newLog), newLog)
s.changeService.Submit(ctx, models.ChangeModelPost, "add", true, changekeys.Listed(newLog, posts), newLog, posts)
results = append(results, newLog)
}
}
default:
{
return nil, errors.New("Invalid importer: " + importer.String())
}
}
return results, nil
}
func (s *LogService) Update(ctx context.Context, id string, update models.LogUpdate) (*models.Log, error) {
log, err := s.logs.Find(ctx, id)
if err != nil {
return nil, err
}
if err := s.authService.CheckPermission(ctx, "edit", log); err != nil {
return nil, err
}
log, err = s.logs.Update(ctx, *log, update)
if err != nil {
return nil, err
}
s.changeService.Submit(ctx, models.ChangeModelLog, "edit", true, changekeys.Listed(log), log)
return log, nil
}
func (s *LogService) SplitLog(ctx context.Context, logId string, startPostId string) (*models.Log, error) {
// Find log
l, err := s.logs.Find(ctx, logId)
if err != nil {
return nil, err
}
if err := s.authService.CheckPermission(ctx, "add", l); err != nil {
return nil, err
}
// Find posts
posts, err := s.ListPosts(ctx, &models.PostFilter{LogID: &l.ShortID})
if err != nil {
return nil, err
}
if len(posts) == 0 {
return nil, errors.New("cannot split empty log")
}
// Cut the posts slice.
firstPost := posts[0]
cutPosts := posts[len(posts):]
for i, post := range posts {
if post.ID == startPostId {
cutPosts = posts[i:]
firstPost = post
break
}
}
if len(cutPosts) == 0 {
return nil, errors.New("post not found")
}
if len(cutPosts) == len(posts) || firstPost.Time.Equal(l.Date) {
return nil, errors.New("cannot move posts")
}
// Create a new log
newLog := &models.Log{
Date: firstPost.Time,
ChannelName: l.ChannelName,
EventName: l.EventName,
}
newLog, err = s.logs.Insert(ctx, *newLog)
if err != nil {
return nil, err
}
// Put the cut posts in the new log
newPosts := make([]*models.Post, 0, len(cutPosts))
for _, post := range cutPosts {
postCopy := *post
postCopy.ID = ""
postCopy.LogID = newLog.ShortID
newPost, err := s.posts.Insert(ctx, postCopy)
if err != nil {
_ = s.logs.Delete(ctx, *newLog)
return nil, err
}
newPosts = append(newPosts, newPost)
}
// Remove the posts from the old log (this can't error because that'll make a mess)
for _, post := range cutPosts {
err := s.posts.Delete(ctx, *post)
if err != nil {
log.Printf("Failed to delete post %s: %s", post.ID, err)
}
}
// Submit the changes
s.changeService.Submit(ctx, models.ChangeModelPost, "remove", true, changekeys.Many(l, cutPosts), cutPosts)
s.changeService.Submit(ctx, models.ChangeModelLog, "add", true, changekeys.Listed(newLog), newLog)
s.changeService.Submit(ctx, models.ChangeModelPost, "add", true, changekeys.Many(newLog, newPosts), newPosts)
// Refresh character lists.
_, _ = s.refreshLogCharacters(ctx, *l, nil, false)
newLog2, err := s.refreshLogCharacters(ctx, *newLog, nil, false)
if err == nil {
newLog = newLog2
}
return newLog, nil
}
func (s *LogService) MergeLogs(ctx context.Context, targetID string, sourceID string, removeAfter bool) (*models.Log, error) {
// Check permissions
if err := s.authService.CheckPermission(ctx, "edit", &models.Log{}); err != nil {
return nil, err
}
if removeAfter {
if err := s.authService.CheckPermission(ctx, "remove", &models.Log{}); err != nil {
return nil, err
}
}
// Merge log posts into log.
source, err := s.logs.Find(ctx, sourceID)
if err != nil {
return nil, errors.New("could not find source log: " + err.Error())
}
target, err := s.logs.Find(ctx, targetID)
if err != nil {
return nil, errors.New("could not find target log: " + err.Error())
}
// Get the source posts.
posts, err := s.posts.List(ctx, models.PostFilter{LogID: &source.ShortID})
if err != nil {
return nil, errors.New("could not fetch source posts: " + err.Error())
}
// Associate the posts with the target logs
for _, post := range posts {
post.ID = ""
post.LogID = target.ShortID
}
// Insert them
posts, err = s.posts.InsertMany(ctx, posts...)
if err != nil {
return nil, errors.New("could not insert posts into target: " + err.Error())
}
// Remove other log
if removeAfter {
err = s.logs.Delete(ctx, *source)
if err != nil {
return nil, errors.New("posts have been inserted, but could not remove source: " + err.Error())
}
s.changeService.Submit(ctx, models.ChangeModelLog, "remove", true, changekeys.Listed(source), source)
}
// Refresh characters
target2, err := s.refreshLogCharacters(ctx, *target, nil, false)
if err != nil {
log.Printf("Failed to update characters in log %s: %s", target.ID, err)
} else {
target = target2
}
// Submit changes after the target
s.changeService.Submit(ctx, models.ChangeModelPost, "add", true, changekeys.Many(target, posts), target, posts)
return target, nil
}
func (s *LogService) AddPost(ctx context.Context, logId string, time time.Time, kind, nick, text string) (*models.Post, error) {
if kind == "" || nick == "" || time.IsZero() {
return nil, errors.New("kind, nick and time must be non-empty")
}
l, err := s.logs.Find(ctx, logId)
if err != nil {
return nil, err
}
post := &models.Post{
LogID: l.ShortID,
Kind: kind,
Nick: nick,
Text: text,
Time: time,
}
if err := s.authService.CheckPermission(ctx, "add", post); err != nil {
return nil, err
}
post, err = s.posts.Insert(ctx, *post)
if err != nil {
return nil, err
}
l2, err := s.refreshLogCharacters(ctx, *l, nil, false)
if err != nil {
log.Printf("Failed to update characters in log %s: %s", l.ID, err)
} else {
l = l2
}
s.changeService.Submit(ctx, models.ChangeModelPost, "add", true, changekeys.Many(l, post), post)
return post, nil
}
func (s *LogService) EditPost(ctx context.Context, id string, update models.PostUpdate) (*models.Post, error) {
if (update.Kind != nil && *update.Kind == "") || (update.Nick != nil && *update.Nick == "") || (update.Text != nil && *update.Text == "") {
return nil, errors.New("kind, nick and time must be non-empty")
}
post, err := s.posts.Find(ctx, id)
if err != nil {
return nil, err
}
if err := s.authService.CheckPermission(ctx, "edit", post); err != nil {
return nil, err
}
post, err = s.posts.Update(ctx, *post, update)
if err != nil {
return nil, err
}
go func() {
l, err := s.logs.Find(context.Background(), post.LogID)
if err != nil {
return
}
_, err = s.refreshLogCharacters(ctx, *l, nil, false)
if err != nil {
log.Printf("Failed to update characters in log %s: %s", l.ID, err)
}
s.changeService.Submit(ctx, models.ChangeModelPost, "edit", true, changekeys.Many(l, post), post)
}()
return post, nil
}
func (s *LogService) MovePost(ctx context.Context, id string, position int) ([]*models.Post, error) {
if position < 1 {
return nil, repositories.ErrInvalidPosition
}
post, err := s.posts.Find(ctx, id)
if err != nil {
return nil, err
}
if err := s.authService.CheckPermission(ctx, "move", post); err != nil {
return nil, err
}
posts, err := s.posts.Move(ctx, *post, position)
if err != nil {
return nil, err
}
go func() {
if len(posts) == 0 {
return
}
log, err := s.logs.Find(context.Background(), posts[0].LogID)
if err != nil {
return
}
s.changeService.Submit(ctx, models.ChangeModelPost, "move", true, changekeys.Many(log, posts), posts)
}()
return posts, nil
}
func (s *LogService) DeletePost(ctx context.Context, id string) (*models.Post, error) {
post, err := s.posts.Find(ctx, id)
if err != nil {
return nil, err
}
if err := s.authService.CheckPermission(ctx, "remove", post); err != nil {
return nil, err
}
err = s.posts.Delete(ctx, *post)
if err != nil {
return nil, err
}
go func() {
l, err := s.logs.Find(context.Background(), post.LogID)
if err != nil {
return
}
_, err = s.refreshLogCharacters(ctx, *l, nil, false)
if err != nil {
log.Printf("Failed to update characters in log %s: %s", l.ID, err)
}
s.changeService.Submit(ctx, models.ChangeModelPost, "remove", true, changekeys.Many(l, post), post)
}()
return post, nil
}
func (s *LogService) Delete(ctx context.Context, id string) (*models.Log, error) {
log, err := s.logs.Find(ctx, id)
if err != nil {
return nil, err
}
if err := s.authService.CheckPermission(ctx, "remove", log); err != nil {
return nil, err
}
err = s.logs.Delete(ctx, *log)
if err != nil {
return nil, err
}
s.changeService.Submit(ctx, models.ChangeModelLog, "remove", true, changekeys.Listed(log), log)
return log, nil
}
func (s *LogService) FixImportDateBug(ctx context.Context) error {
start := time.Now()
logs, err := s.logs.List(ctx, models.LogFilter{})
if err != nil {
return err
}
eg := errgroup.Group{}
for i := range logs {
l := logs[i]
eg.Go(func() error {
return s.fixImportDateBug(ctx, *l)
})
}
err = eg.Wait()
if err != nil {
return err
}
log.Printf("Date import bug check finished: logs: %d, duration: %s", len(logs), time.Since(start))
return nil
}
func (s *LogService) fixImportDateBug(ctx context.Context, l models.Log) error {
// Find the log's posts.
posts, err := s.ListPosts(ctx, &models.PostFilter{LogID: &l.ShortID})
if err != nil {
return err
}
if len(posts) < 8 {
return nil
}
// Find first action post
first := posts[0]
fi := 0
for first.Kind != "action" && first.Kind != "text" {
fi++
if fi >= len(posts) {
return nil
}
first = posts[fi]
}
last := posts[len(posts)-1]
if first == last {
return nil
}
// Stop here if this log probably isn't affected
if last.Time.Sub(first.Time) < time.Hour*72 {
return nil
}
// Find the first post past midnight.
midnight := first
mi := fi
for i, post := range posts[fi+1:] {
if post.Time.Hour() < first.Time.Hour() {
midnight = post
mi = fi + 1 + i
break
}
}
if midnight == last {
return nil
}
if len(posts[mi+1:]) == 1 {
return nil
}
hits := 0
prev := midnight
for _, offender := range posts[mi+1:] {
if offender.Time.Day() != prev.Time.Day() {
hits += 1
}
prev = offender
}
if hits < ((len(posts[mi+1:]) * 3) / 4) {
return nil
}
for _, offender := range posts[mi+1:] {
ot := offender.Time.UTC()
mt := midnight.Time.UTC()
y, m, d := mt.Date()
hr, mi, se, ns := ot.Hour(), ot.Minute(), ot.Second(), ot.Nanosecond()
newTime := time.Date(y, m, d, hr, mi, se, ns, time.UTC)
_, err := s.posts.Update(ctx, *offender, models.PostUpdate{Time: &newTime})
if err != nil {
return err
}
}
log.Printf("Fixed import date bug in %d posts in log %s", len(posts[mi+1:]), l.ID)
return nil
}
func (s *LogService) RefreshAllLogCharacters(ctx context.Context) error {
start := time.Now()
// Get all logs
logs, err := s.logs.List(ctx, models.LogFilter{})
if err != nil {
return err
}
// Check all characters now instead of later.
characters, err := s.characterService.List(ctx, models.CharacterFilter{})
if err != nil {
return err
}
characterMap := s.makeCharacterMap(characters)
s.unknownNicksMutex.Lock()
for key := range s.unknownNicks {
delete(s.unknownNicks, key)
}
s.unknownNicksMutex.Unlock()
tokens := make(chan struct{}, 33)
for i := 0; i < 32; i++ {
tokens <- struct{}{}
}
eg := errgroup.Group{}
for i := range logs {
l := logs[i]
eg.Go(func() error {
<-tokens
defer func() { tokens <- struct{}{} }()
_, err := s.refreshLogCharacters(ctx, *l, characterMap, true)
return err
})
}
err = eg.Wait()
if err != nil {
return err
}
s.unknownNicksMutex.Lock()
unknownCount := len(s.unknownNicks)
s.unknownNicksMutex.Unlock()
log.Printf("Full log character refresh complete; nicks: %d, unknowns: %d, logs: %d, duration: %s", len(characterMap), unknownCount, len(logs), time.Since(start))
return nil
}
func (s *LogService) RefreshLogCharacters(ctx context.Context, log models.Log) (*models.Log, error) {
return s.refreshLogCharacters(ctx, log, nil, false)
}
func (s *LogService) refreshLogCharacters(ctx context.Context, log models.Log, characterMap map[string]*models.Character, useUnknownNicks bool) (*models.Log, error) {
posts, err := s.ListPosts(ctx, &models.PostFilter{LogID: &log.ShortID})
if err != nil {
return nil, err
}
counts := make(map[string]int)
added := make(map[string]bool)
removed := make(map[string]bool)
for _, post := range posts {
if post.Kind == "text" || post.Kind == "action" {
if strings.HasPrefix(post.Text, "(") || strings.Contains(post.Nick, "(") || strings.Contains(post.Nick, "[E]") || strings.HasSuffix(post.Nick, "|") {
continue
}
// Clean up the nick (remove possessive suffix, comma, formatting stuff)
if strings.HasSuffix(post.Nick, "'s") || strings.HasSuffix(post.Nick, "`s") {
post.Nick = post.Nick[:len(post.Nick)-2]
} else if strings.HasSuffix(post.Nick, "'") || strings.HasSuffix(post.Nick, "`") || strings.HasSuffix(post.Nick, ",") || strings.HasSuffix(post.Nick, "\x0f") {
post.Nick = post.Nick[:len(post.Nick)-1]
}
added[post.Nick] = true
counts[post.Nick]++
}
if post.Kind == "chars" {
tokens := strings.Fields(post.Text)
for _, token := range tokens {
if strings.HasPrefix(token, "-") {
removed[token[1:]] = true
} else {
added[strings.Replace(token, "+", "", 1)] = true
}
}
}
}
nicks := make([]string, 0, len(added))
for nick := range added {
if added[nick] && !removed[nick] {
nicks = append(nicks, nick)
}
}
if characterMap == nil {
characters, err := s.characterService.List(ctx, models.CharacterFilter{Nicks: nicks})
if err != nil {
return nil, err
}
characterMap = s.makeCharacterMap(characters)
}
log.CharacterIDs = log.CharacterIDs[:0]
for key := range added {
delete(added, key)
}
unknowned := make(map[string]bool)
for _, nick := range nicks {
character := characterMap[nick]
if character == nil {
if useUnknownNicks && !unknowned[nick] {
unknowned[nick] = true
s.unknownNicksMutex.Lock()
s.unknownNicks[nick]++
s.unknownNicksMutex.Unlock()
}
continue
} else if added[character.ID] {
continue
}
added[character.ID] = true
log.CharacterIDs = append(log.CharacterIDs, character.ID)
}
return s.logs.Update(ctx, log, models.LogUpdate{CharacterIDs: log.CharacterIDs})
}
func (s *LogService) makeCharacterMap(characters []*models.Character) map[string]*models.Character {
characterMap := make(map[string]*models.Character, len(characters)*3)
for _, character := range characters {
for _, nick := range character.Nicks {
characterMap[nick] = character
}
}
return characterMap
}
func (s *LogService) NextLogs(ctx context.Context, log *models.Log) ([]*models.LogSuggestion, error) {
minDate := log.Date.Add(time.Millisecond)
logs, err := s.logs.List(ctx, models.LogFilter{
MinDate: &minDate,
})
if err != nil {
return nil, err
}
sort.Slice(logs, func(i, j int) bool {
return logs[i].Date.Before(logs[j].Date)
})
return s.findSuggestions(ctx, log, logs)
}
func (s *LogService) PrevLogs(ctx context.Context, log *models.Log) ([]*models.LogSuggestion, error) {
logs, err := s.logs.List(ctx, models.LogFilter{
MaxDate: &log.Date,
})
if err != nil {
return nil, err
}
return s.findSuggestions(ctx, log, logs)
}
func (s *LogService) findSuggestions(ctx context.Context, log *models.Log, logs []*models.Log) ([]*models.LogSuggestion, error) {
characters, err := s.characterService.List(ctx, models.CharacterFilter{
IDs: log.CharacterIDs,
})
if err != nil {
return nil, err
}
charIntersect := func(l1, l2 *models.Log) []*models.Character {
results := make([]*models.Character, 0, len(l1.CharacterIDs))
for _, c1 := range characters {
for _, c2ID := range l2.CharacterIDs {
if c1.ID == c2ID {
results = append(results, c1)
break
}
}
}
return results
}
groupKey := func(characters []*models.Character) string {
if len(characters) == 0 {
return ""
} else if len(characters) == 1 {
return characters[0].ID
}
builder := strings.Builder{}
builder.WriteString(characters[0].ID)
for _, character := range characters {
builder.WriteRune(',')
builder.WriteString(character.ID)
}
return builder.String()
}
suggestions := make([]*models.LogSuggestion, 0, 16)
foundGroups := make(map[string]bool)
foundChannel := false
for _, log2 := range logs {
hasEvent := log.EventName != "" && log2.EventName == log.EventName
hasChannel := log.ChannelName == log2.ChannelName
characters := charIntersect(log, log2)
groupKey := groupKey(characters)
suggestion := &models.LogSuggestion{
Log: log2,
Characters: characters,
HasChannel: hasChannel,
HasEvent: hasEvent,
}
if hasChannel && foundChannel {
foundChannel = true
foundGroups[groupKey] = true
suggestions = append(suggestions, suggestion)
} else if hasEvent {
foundGroups[groupKey] = true
suggestions = append(suggestions, suggestion)
} else if len(suggestions) < 8 && !hasEvent && len(characters) > 1 && !foundGroups[groupKey] {
foundGroups[groupKey] = true
suggestions = append(suggestions, suggestion)
}
}
return suggestions, nil
}