|
|
package services
import ( "context" "errors" "git.aiterp.net/rpdata/api/models" "git.aiterp.net/rpdata/api/models/changekeys" "git.aiterp.net/rpdata/api/repositories" "git.aiterp.net/rpdata/api/services/parsers" "golang.org/x/sync/errgroup" "log" "sort" "strings" "sync" "time" )
type LogService struct { logs repositories.LogRepository posts repositories.PostRepository changeService *ChangeService channelService *ChannelService characterService *CharacterService authService *AuthService
unknownNicks map[string]int unknownNicksMutex sync.Mutex }
func (s *LogService) Find(ctx context.Context, id string) (*models.Log, error) { return s.logs.Find(ctx, id) }
func (s *LogService) FindPosts(ctx context.Context, id string) (*models.Post, error) { return s.posts.Find(ctx, id) }
func (s *LogService) List(ctx context.Context, filter *models.LogFilter) ([]*models.Log, error) { if filter == nil { filter = &models.LogFilter{} }
return s.logs.List(ctx, *filter) }
func (s *LogService) ListPosts(ctx context.Context, filter *models.PostFilter) ([]*models.Post, error) { // Some sanity checks to avoid querying an insame amount of posts.
if filter == nil { filter = &models.PostFilter{Limit: 100} } else { if (filter.Limit <= 0 || filter.Limit > 512) && (filter.LogID == nil && len(filter.IDs) == 0) { return nil, errors.New("a limit of 0 (no limit) or >512 without a logId or a set of IDs is not allowed") }
if len(filter.IDs) > 100 { return nil, errors.New("you may not query for more than 100 ids, split your query") } }
return s.posts.List(ctx, *filter) }
func (s *LogService) ListUnknownNicks() []*models.UnknownNick { s.unknownNicksMutex.Lock() nicks := make([]*models.UnknownNick, 0, len(s.unknownNicks)) for nick, score := range s.unknownNicks { nicks = append(nicks, &models.UnknownNick{ Nick: nick, Score: score, }) } s.unknownNicksMutex.Unlock()
sort.Slice(nicks, func(i, j int) bool { return nicks[i].Score > nicks[j].Score })
return nicks }
func (s *LogService) Create(ctx context.Context, title, description, channelName, eventName string, open bool) (*models.Log, error) { if channelName == "" { return nil, errors.New("channel name cannot be empty") }
log := &models.Log{ Title: title, Description: description, ChannelName: channelName, EventName: eventName, Date: time.Now(), Open: open, }
if err := s.authService.CheckPermission(ctx, "add", log); err != nil { return nil, err }
_, err := s.channelService.Ensure(ctx, channelName) if err != nil { return nil, err }
log, err = s.logs.Insert(ctx, *log) if err != nil { return nil, err }
s.changeService.Submit(ctx, models.ChangeModelLog, "add", true, changekeys.Listed(log), log)
return log, nil }
// Import creates new logs from common formats.
func (s *LogService) Import(ctx context.Context, importer models.LogImporter, date time.Time, tz *time.Location, channelName string, sessionThreshold time.Duration, data string) ([]*models.Log, error) { if err := s.authService.CheckPermission(ctx, "add", &models.Log{}); err != nil { return nil, err }
results := make([]*models.Log, 0, 8)
_, err := s.channelService.Ensure(ctx, channelName) if err != nil { return nil, err }
eventName := "" if channel, err := s.channelService.Find(ctx, channelName); err == nil { eventName = channel.EventName }
date = date.In(tz)
switch importer { case models.LogImporterMircLike: { if date.IsZero() { return nil, errors.New("date is not optional for mirc-like logs") }
parsed, err := parsers.MircLog(data, date, true) if err != nil { return nil, err } parsed.Log.EventName = eventName parsed.Log.ChannelName = channelName
newLog, err := s.logs.Insert(ctx, parsed.Log) if err != nil { return nil, err }
for _, post := range parsed.Posts { post.LogID = newLog.ShortID }
posts, err := s.posts.InsertMany(ctx, parsed.Posts...) if err != nil { _ = s.logs.Delete(ctx, *newLog)
return nil, err }
s.changeService.Submit(ctx, models.ChangeModelLog, "add", true, changekeys.Listed(newLog), newLog) s.changeService.Submit(ctx, models.ChangeModelPost, "add", true, changekeys.Listed(newLog, posts), newLog, posts)
refreshedLog, err := s.refreshLogCharacters(ctx, *newLog, nil, false) if err != nil { log.Printf("Failed to update characters in newLog %s: %s", newLog.ID, err) } else { newLog = refreshedLog }
results = append(results, newLog) } case models.LogImporterForumLog: { parseResults, err := parsers.ForumLog(data, tz) if err != nil { return nil, err }
for _, parsed := range parseResults { parsed.Log.EventName = eventName parsed.Log.ChannelName = channelName
newLog, err := s.logs.Insert(ctx, parsed.Log) if err != nil { return nil, err }
for _, post := range parsed.Posts { post.LogID = newLog.ShortID }
posts, err := s.posts.InsertMany(ctx, parsed.Posts...) if err != nil { _ = s.logs.Delete(ctx, *newLog)
return nil, err }
s.changeService.Submit(ctx, models.ChangeModelLog, "add", true, changekeys.Listed(newLog), newLog) s.changeService.Submit(ctx, models.ChangeModelPost, "add", true, changekeys.Listed(newLog, posts), newLog, posts)
results = append(results, newLog) }
} case models.LogImporterIrcCloud: { parseResults, err := parsers.IRCCloudLogs(data, tz, sessionThreshold) if err != nil { return nil, err }
for _, parsed := range parseResults { parsed.Log.EventName = eventName parsed.Log.ChannelName = channelName
newLog, err := s.logs.Insert(ctx, parsed.Log) if err != nil { return nil, err }
for _, post := range parsed.Posts { post.LogID = newLog.ShortID }
posts, err := s.posts.InsertMany(ctx, parsed.Posts...) if err != nil { _ = s.logs.Delete(ctx, *newLog)
return results, err }
refreshedLog, err := s.refreshLogCharacters(ctx, *newLog, nil, false) if err != nil { log.Printf("Failed to update characters in newLog %s: %s", newLog.ID, err) } else { newLog = refreshedLog }
s.changeService.Submit(ctx, models.ChangeModelLog, "add", true, changekeys.Listed(newLog), newLog) s.changeService.Submit(ctx, models.ChangeModelPost, "add", true, changekeys.Listed(newLog, posts), newLog, posts)
results = append(results, newLog) } }
default: { return nil, errors.New("Invalid importer: " + importer.String()) } }
return results, nil }
func (s *LogService) Update(ctx context.Context, id string, update models.LogUpdate) (*models.Log, error) { log, err := s.logs.Find(ctx, id) if err != nil { return nil, err }
if err := s.authService.CheckPermission(ctx, "edit", log); err != nil { return nil, err }
log, err = s.logs.Update(ctx, *log, update) if err != nil { return nil, err }
s.changeService.Submit(ctx, models.ChangeModelLog, "edit", true, changekeys.Listed(log), log)
return log, nil }
func (s *LogService) SplitLog(ctx context.Context, logId string, startPostId string) (*models.Log, error) { // Find log
l, err := s.logs.Find(ctx, logId) if err != nil { return nil, err } if err := s.authService.CheckPermission(ctx, "add", l); err != nil { return nil, err }
// Find posts
posts, err := s.ListPosts(ctx, &models.PostFilter{LogID: &l.ShortID}) if err != nil { return nil, err } if len(posts) == 0 { return nil, errors.New("cannot split empty log") }
// Cut the posts slice.
firstPost := posts[0] cutPosts := posts[len(posts):] for i, post := range posts { if post.ID == startPostId { cutPosts = posts[i:] firstPost = post break } } if len(cutPosts) == 0 { return nil, errors.New("post not found") } if len(cutPosts) == len(posts) || firstPost.Time.Equal(l.Date) { return nil, errors.New("cannot move posts") }
// Create a new log
newLog := &models.Log{ Date: firstPost.Time, ChannelName: l.ChannelName, EventName: l.EventName, } newLog, err = s.logs.Insert(ctx, *newLog) if err != nil { return nil, err }
// Put the cut posts in the new log
newPosts := make([]*models.Post, 0, len(cutPosts)) for _, post := range cutPosts { postCopy := *post postCopy.ID = "" postCopy.LogID = newLog.ShortID
newPost, err := s.posts.Insert(ctx, postCopy) if err != nil { _ = s.logs.Delete(ctx, *newLog) return nil, err }
newPosts = append(newPosts, newPost) }
// Remove the posts from the old log (this can't error because that'll make a mess)
for _, post := range cutPosts { err := s.posts.Delete(ctx, *post) if err != nil { log.Printf("Failed to delete post %s: %s", post.ID, err) } }
// Submit the changes
s.changeService.Submit(ctx, models.ChangeModelPost, "remove", true, changekeys.Many(l, cutPosts), cutPosts) s.changeService.Submit(ctx, models.ChangeModelLog, "add", true, changekeys.Listed(newLog), newLog) s.changeService.Submit(ctx, models.ChangeModelPost, "add", true, changekeys.Many(newLog, newPosts), newPosts)
// Refresh character lists.
_, _ = s.refreshLogCharacters(ctx, *l, nil, false) newLog2, err := s.refreshLogCharacters(ctx, *newLog, nil, false) if err == nil { newLog = newLog2 }
return newLog, nil }
func (s *LogService) MergeLogs(ctx context.Context, targetID string, sourceID string, removeAfter bool) (*models.Log, error) { // Check permissions
if err := s.authService.CheckPermission(ctx, "edit", &models.Log{}); err != nil { return nil, err } if removeAfter { if err := s.authService.CheckPermission(ctx, "remove", &models.Log{}); err != nil { return nil, err } }
// Merge log posts into log.
source, err := s.logs.Find(ctx, sourceID) if err != nil { return nil, errors.New("could not find source log: " + err.Error()) } target, err := s.logs.Find(ctx, targetID) if err != nil { return nil, errors.New("could not find target log: " + err.Error()) }
// Get the source posts.
posts, err := s.posts.List(ctx, models.PostFilter{LogID: &source.ShortID}) if err != nil { return nil, errors.New("could not fetch source posts: " + err.Error()) }
// Associate the posts with the target logs
for _, post := range posts { post.ID = "" post.LogID = target.ShortID }
// Insert them
posts, err = s.posts.InsertMany(ctx, posts...) if err != nil { return nil, errors.New("could not insert posts into target: " + err.Error()) }
// Remove other log
if removeAfter { err = s.logs.Delete(ctx, *source) if err != nil { return nil, errors.New("posts have been inserted, but could not remove source: " + err.Error()) }
s.changeService.Submit(ctx, models.ChangeModelLog, "remove", true, changekeys.Listed(source), source) }
// Refresh characters
target2, err := s.refreshLogCharacters(ctx, *target, nil, false) if err != nil { log.Printf("Failed to update characters in log %s: %s", target.ID, err) } else { target = target2 }
// Submit changes after the target
s.changeService.Submit(ctx, models.ChangeModelPost, "add", true, changekeys.Many(target, posts), target, posts)
return target, nil }
func (s *LogService) AddPost(ctx context.Context, logId string, time time.Time, kind, nick, text string) (*models.Post, error) { if kind == "" || nick == "" || time.IsZero() { return nil, errors.New("kind, nick and time must be non-empty") }
l, err := s.logs.Find(ctx, logId) if err != nil { return nil, err }
post := &models.Post{ LogID: l.ShortID, Kind: kind, Nick: nick, Text: text, Time: time, }
if err := s.authService.CheckPermission(ctx, "add", post); err != nil { return nil, err }
post, err = s.posts.Insert(ctx, *post) if err != nil { return nil, err }
l2, err := s.refreshLogCharacters(ctx, *l, nil, false) if err != nil { log.Printf("Failed to update characters in log %s: %s", l.ID, err) } else { l = l2 }
s.changeService.Submit(ctx, models.ChangeModelPost, "add", true, changekeys.Many(l, post), post)
return post, nil }
func (s *LogService) EditPost(ctx context.Context, id string, update models.PostUpdate) (*models.Post, error) { if (update.Kind != nil && *update.Kind == "") || (update.Nick != nil && *update.Nick == "") || (update.Text != nil && *update.Text == "") { return nil, errors.New("kind, nick and time must be non-empty") }
post, err := s.posts.Find(ctx, id) if err != nil { return nil, err }
if err := s.authService.CheckPermission(ctx, "edit", post); err != nil { return nil, err }
post, err = s.posts.Update(ctx, *post, update) if err != nil { return nil, err }
spinOffCtx := s.authService.SpinOffContext(ctx) go func() { timeout, cancel := context.WithTimeout(spinOffCtx, time.Minute*10) defer cancel()
l, err := s.logs.Find(timeout, post.LogID) if err != nil { return }
s.changeService.Submit(timeout, models.ChangeModelPost, "edit", true, changekeys.Many(l, post), post)
_, err = s.refreshLogCharacters(timeout, *l, nil, false) if err != nil { log.Printf("Failed to update characters in log %s: %s", l.ID, err) } }()
return post, nil }
func (s *LogService) MovePost(ctx context.Context, id string, position int) ([]*models.Post, error) { if position < 1 { return nil, repositories.ErrInvalidPosition }
post, err := s.posts.Find(ctx, id) if err != nil { return nil, err }
if err := s.authService.CheckPermission(ctx, "move", post); err != nil { return nil, err }
posts, err := s.posts.Move(ctx, *post, position) if err != nil { return nil, err }
go func() { if len(posts) == 0 { return }
log, err := s.logs.Find(context.Background(), posts[0].LogID) if err != nil { return }
s.changeService.Submit(ctx, models.ChangeModelPost, "move", true, changekeys.Many(log, posts), posts) }()
return posts, nil }
func (s *LogService) DeletePost(ctx context.Context, id string) (*models.Post, error) { post, err := s.posts.Find(ctx, id) if err != nil { return nil, err }
if err := s.authService.CheckPermission(ctx, "remove", post); err != nil { return nil, err }
err = s.posts.Delete(ctx, *post) if err != nil { return nil, err }
go func() { l, err := s.logs.Find(context.Background(), post.LogID) if err != nil { return }
_, err = s.refreshLogCharacters(ctx, *l, nil, false) if err != nil { log.Printf("Failed to update characters in log %s: %s", l.ID, err) }
s.changeService.Submit(ctx, models.ChangeModelPost, "remove", true, changekeys.Many(l, post), post) }()
return post, nil }
func (s *LogService) Delete(ctx context.Context, id string) (*models.Log, error) { log, err := s.logs.Find(ctx, id) if err != nil { return nil, err }
if err := s.authService.CheckPermission(ctx, "remove", log); err != nil { return nil, err }
err = s.logs.Delete(ctx, *log) if err != nil { return nil, err }
s.changeService.Submit(ctx, models.ChangeModelLog, "remove", true, changekeys.Listed(log), log)
return log, nil }
func (s *LogService) FixImportDateBug(ctx context.Context) error { start := time.Now()
logs, err := s.logs.List(ctx, models.LogFilter{}) if err != nil { return err }
eg := errgroup.Group{} for i := range logs { l := logs[i]
eg.Go(func() error { return s.fixImportDateBug(ctx, *l) }) } err = eg.Wait() if err != nil { return err }
log.Printf("Date import bug check finished: logs: %d, duration: %s", len(logs), time.Since(start))
return nil }
func (s *LogService) fixImportDateBug(ctx context.Context, l models.Log) error { // Find the log's posts.
posts, err := s.ListPosts(ctx, &models.PostFilter{LogID: &l.ShortID}) if err != nil { return err } if len(posts) < 8 { return nil }
// Find first action post
first := posts[0] fi := 0 for first.Kind != "action" && first.Kind != "text" { fi++ if fi >= len(posts) { return nil }
first = posts[fi] } last := posts[len(posts)-1] if first == last { return nil }
// Stop here if this log probably isn't affected
if last.Time.Sub(first.Time) < time.Hour*72 { return nil }
// Find the first post past midnight.
midnight := first mi := fi for i, post := range posts[fi+1:] { if post.Time.Hour() < first.Time.Hour() { midnight = post mi = fi + 1 + i break } } if midnight == last { return nil }
if len(posts[mi+1:]) == 1 { return nil }
hits := 0 prev := midnight for _, offender := range posts[mi+1:] { if offender.Time.Day() != prev.Time.Day() { hits += 1 }
prev = offender } if hits < ((len(posts[mi+1:]) * 3) / 4) { return nil }
for _, offender := range posts[mi+1:] { ot := offender.Time.UTC() mt := midnight.Time.UTC()
y, m, d := mt.Date() hr, mi, se, ns := ot.Hour(), ot.Minute(), ot.Second(), ot.Nanosecond()
newTime := time.Date(y, m, d, hr, mi, se, ns, time.UTC)
_, err := s.posts.Update(ctx, *offender, models.PostUpdate{Time: &newTime}) if err != nil { return err } }
log.Printf("Fixed import date bug in %d posts in log %s", len(posts[mi+1:]), l.ID)
return nil }
func (s *LogService) RefreshAllLogCharacters(ctx context.Context) error { start := time.Now()
// Get all logs
logs, err := s.logs.List(ctx, models.LogFilter{}) if err != nil { return err }
// Check all characters now instead of later.
characters, err := s.characterService.List(ctx, models.CharacterFilter{}) if err != nil { return err } characterMap := s.makeCharacterMap(characters)
s.unknownNicksMutex.Lock() for key := range s.unknownNicks { delete(s.unknownNicks, key) } s.unknownNicksMutex.Unlock()
tokens := make(chan struct{}, 33) for i := 0; i < 32; i++ { tokens <- struct{}{} }
eg := errgroup.Group{} for i := range logs { l := logs[i]
eg.Go(func() error { <-tokens defer func() { tokens <- struct{}{} }()
_, err := s.refreshLogCharacters(ctx, *l, characterMap, true) return err }) } err = eg.Wait() if err != nil { return err }
s.unknownNicksMutex.Lock() unknownCount := len(s.unknownNicks) s.unknownNicksMutex.Unlock()
log.Printf("Full log character refresh complete; nicks: %d, unknowns: %d, logs: %d, duration: %s", len(characterMap), unknownCount, len(logs), time.Since(start))
return nil }
func (s *LogService) RefreshLogCharacters(ctx context.Context, log models.Log) (*models.Log, error) { return s.refreshLogCharacters(ctx, log, nil, false) }
func (s *LogService) refreshLogCharacters(ctx context.Context, log models.Log, characterMap map[string]*models.Character, useUnknownNicks bool) (*models.Log, error) { posts, err := s.ListPosts(ctx, &models.PostFilter{LogID: &log.ShortID}) if err != nil { return nil, err }
counts := make(map[string]int) added := make(map[string]bool) removed := make(map[string]bool) for _, post := range posts { if post.Kind == "text" || post.Kind == "action" { if strings.HasPrefix(post.Text, "(") || strings.Contains(post.Nick, "(") || strings.Contains(post.Nick, "[E]") || strings.HasSuffix(post.Nick, "|") { continue }
// Clean up the nick (remove possessive suffix, comma, formatting stuff)
if strings.HasSuffix(post.Nick, "'s") || strings.HasSuffix(post.Nick, "`s") { post.Nick = post.Nick[:len(post.Nick)-2] } else if strings.HasSuffix(post.Nick, "'") || strings.HasSuffix(post.Nick, "`") || strings.HasSuffix(post.Nick, ",") || strings.HasSuffix(post.Nick, "\x0f") { post.Nick = post.Nick[:len(post.Nick)-1] }
added[post.Nick] = true counts[post.Nick]++ } if post.Kind == "chars" { tokens := strings.Fields(post.Text) for _, token := range tokens { if strings.HasPrefix(token, "-") { removed[token[1:]] = true } else { added[strings.Replace(token, "+", "", 1)] = true } } } }
nicks := make([]string, 0, len(added)) for nick := range added { if added[nick] && !removed[nick] { nicks = append(nicks, nick) } }
if characterMap == nil { characters, err := s.characterService.List(ctx, models.CharacterFilter{Nicks: nicks}) if err != nil { return nil, err }
characterMap = s.makeCharacterMap(characters) }
log.CharacterIDs = log.CharacterIDs[:0] for key := range added { delete(added, key) } unknowned := make(map[string]bool) for _, nick := range nicks { character := characterMap[nick] if character == nil { if useUnknownNicks && !unknowned[nick] { unknowned[nick] = true s.unknownNicksMutex.Lock() s.unknownNicks[nick]++ s.unknownNicksMutex.Unlock() }
continue } else if added[character.ID] { continue } added[character.ID] = true
log.CharacterIDs = append(log.CharacterIDs, character.ID) }
return s.logs.Update(ctx, log, models.LogUpdate{CharacterIDs: log.CharacterIDs}) }
func (s *LogService) makeCharacterMap(characters []*models.Character) map[string]*models.Character { characterMap := make(map[string]*models.Character, len(characters)*3) for _, character := range characters { for _, nick := range character.Nicks { characterMap[nick] = character } }
return characterMap }
func (s *LogService) NextLogs(ctx context.Context, log *models.Log) ([]*models.LogSuggestion, error) { minDate := log.Date.Add(time.Millisecond) logs, err := s.logs.List(ctx, models.LogFilter{ MinDate: &minDate, }) if err != nil { return nil, err }
sort.Slice(logs, func(i, j int) bool { return logs[i].Date.Before(logs[j].Date) })
if len(logs) >= 1 && logs[0].ID == log.ID { logs = logs[1:] }
return s.findSuggestions(ctx, log, logs) }
func (s *LogService) PrevLogs(ctx context.Context, log *models.Log) ([]*models.LogSuggestion, error) { logs, err := s.logs.List(ctx, models.LogFilter{ MaxDate: &log.Date, }) if err != nil { return nil, err }
if len(logs) >= 1 && logs[0].ID == log.ID { logs = logs[1:] }
return s.findSuggestions(ctx, log, logs) }
func (s *LogService) findSuggestions(ctx context.Context, log *models.Log, logs []*models.Log) ([]*models.LogSuggestion, error) { characters, err := s.characterService.List(ctx, models.CharacterFilter{ IDs: log.CharacterIDs, }) if err != nil { return nil, err }
charIntersect := func(l1, l2 *models.Log) []*models.Character { results := make([]*models.Character, 0, len(l1.CharacterIDs)) for _, c1 := range characters { for _, c2ID := range l2.CharacterIDs { if c1.ID == c2ID { results = append(results, c1) break } } }
return results } groupKey := func(characters []*models.Character) string { if len(characters) == 0 { return "" } else if len(characters) == 1 { return characters[0].ID }
builder := strings.Builder{} builder.WriteString(characters[0].ID)
for _, character := range characters { builder.WriteRune(',') builder.WriteString(character.ID) }
return builder.String() }
suggestions := make([]*models.LogSuggestion, 0, 16) foundGroups := make(map[string]bool) foundChannel := false
for _, log2 := range logs { hasEvent := log.EventName != "" && log2.EventName == log.EventName hasChannel := log.ChannelName == log2.ChannelName characters := charIntersect(log, log2) groupKey := groupKey(characters)
suggestion := &models.LogSuggestion{ Log: log2, Characters: characters, HasChannel: hasChannel, HasEvent: hasEvent, }
if hasChannel && foundChannel { foundChannel = true foundGroups[groupKey] = true suggestions = append(suggestions, suggestion) } else if hasEvent { foundGroups[groupKey] = true suggestions = append(suggestions, suggestion) } else if len(suggestions) < 8 && !hasEvent && len(characters) > 1 && !foundGroups[groupKey] { foundGroups[groupKey] = true suggestions = append(suggestions, suggestion) } }
return suggestions, nil }
|