package services import ( "context" "errors" "git.aiterp.net/rpdata/api/models" "git.aiterp.net/rpdata/api/models/changekeys" "git.aiterp.net/rpdata/api/repositories" "git.aiterp.net/rpdata/api/services/parsers" "golang.org/x/sync/errgroup" "log" "sort" "strings" "sync" "time" ) type LogService struct { logs repositories.LogRepository posts repositories.PostRepository changeService *ChangeService channelService *ChannelService characterService *CharacterService authService *AuthService unknownNicks map[string]int unknownNicksMutex sync.Mutex } func (s *LogService) Find(ctx context.Context, id string) (*models.Log, error) { return s.logs.Find(ctx, id) } func (s *LogService) FindPosts(ctx context.Context, id string) (*models.Post, error) { return s.posts.Find(ctx, id) } func (s *LogService) List(ctx context.Context, filter *models.LogFilter) ([]*models.Log, error) { if filter == nil { filter = &models.LogFilter{} } return s.logs.List(ctx, *filter) } func (s *LogService) ListPosts(ctx context.Context, filter *models.PostFilter) ([]*models.Post, error) { // Some sanity checks to avoid querying an insame amount of posts. if filter == nil { filter = &models.PostFilter{Limit: 100} } else { if (filter.Limit <= 0 || filter.Limit > 512) && (filter.LogID == nil && len(filter.IDs) == 0) { return nil, errors.New("a limit of 0 (no limit) or >512 without a logId or a set of IDs is not allowed") } if len(filter.IDs) > 100 { return nil, errors.New("you may not query for more than 100 ids, split your query") } } return s.posts.List(ctx, *filter) } func (s *LogService) ListUnknownNicks() []*models.UnknownNick { s.unknownNicksMutex.Lock() nicks := make([]*models.UnknownNick, 0, len(s.unknownNicks)) for nick, score := range s.unknownNicks { nicks = append(nicks, &models.UnknownNick{ Nick: nick, Score: score, }) } s.unknownNicksMutex.Unlock() sort.Slice(nicks, func(i, j int) bool { return nicks[i].Score > nicks[j].Score }) return nicks } func (s *LogService) Create(ctx context.Context, title, description, channelName, eventName string, open bool) (*models.Log, error) { if channelName == "" { return nil, errors.New("channel name cannot be empty") } log := &models.Log{ Title: title, Description: description, ChannelName: channelName, EventName: eventName, Date: time.Now(), Open: open, } if err := s.authService.CheckPermission(ctx, "add", log); err != nil { return nil, err } _, err := s.channelService.Ensure(ctx, channelName) if err != nil { return nil, err } log, err = s.logs.Insert(ctx, *log) if err != nil { return nil, err } s.changeService.Submit(ctx, models.ChangeModelLog, "add", true, changekeys.Listed(log), log) return log, nil } // Import creates new logs from common formats. func (s *LogService) Import(ctx context.Context, importer models.LogImporter, date time.Time, tz *time.Location, channelName string, sessionThreshold time.Duration, data string) ([]*models.Log, error) { if err := s.authService.CheckPermission(ctx, "add", &models.Log{}); err != nil { return nil, err } results := make([]*models.Log, 0, 8) _, err := s.channelService.Ensure(ctx, channelName) if err != nil { return nil, err } eventName := "" if channel, err := s.channelService.Find(ctx, channelName); err == nil { eventName = channel.EventName } date = date.In(tz) switch importer { case models.LogImporterMircLike: { if date.IsZero() { return nil, errors.New("date is not optional for mirc-like logs") } parsed, err := parsers.MircLog(data, date, true) if err != nil { return nil, err } parsed.Log.EventName = eventName parsed.Log.ChannelName = channelName newLog, err := s.logs.Insert(ctx, parsed.Log) if err != nil { return nil, err } for _, post := range parsed.Posts { post.LogID = newLog.ShortID } posts, err := s.posts.InsertMany(ctx, parsed.Posts...) if err != nil { _ = s.logs.Delete(ctx, *newLog) return nil, err } s.changeService.Submit(ctx, models.ChangeModelLog, "add", true, changekeys.Listed(newLog), newLog) s.changeService.Submit(ctx, models.ChangeModelPost, "add", true, changekeys.Listed(newLog, posts), newLog, posts) refreshedLog, err := s.refreshLogCharacters(ctx, *newLog, nil, false) if err != nil { log.Printf("Failed to update characters in newLog %s: %s", newLog.ID, err) } else { newLog = refreshedLog } results = append(results, newLog) } case models.LogImporterForumLog: { parseResults, err := parsers.ForumLog(data, tz) if err != nil { return nil, err } for _, parsed := range parseResults { parsed.Log.EventName = eventName parsed.Log.ChannelName = channelName newLog, err := s.logs.Insert(ctx, parsed.Log) if err != nil { return nil, err } for _, post := range parsed.Posts { post.LogID = newLog.ShortID } posts, err := s.posts.InsertMany(ctx, parsed.Posts...) if err != nil { _ = s.logs.Delete(ctx, *newLog) return nil, err } s.changeService.Submit(ctx, models.ChangeModelLog, "add", true, changekeys.Listed(newLog), newLog) s.changeService.Submit(ctx, models.ChangeModelPost, "add", true, changekeys.Listed(newLog, posts), newLog, posts) results = append(results, newLog) } } case models.LogImporterIrcCloud: { parseResults, err := parsers.IRCCloudLogs(data, tz, sessionThreshold) if err != nil { return nil, err } for _, parsed := range parseResults { parsed.Log.EventName = eventName parsed.Log.ChannelName = channelName newLog, err := s.logs.Insert(ctx, parsed.Log) if err != nil { return nil, err } for _, post := range parsed.Posts { post.LogID = newLog.ShortID } posts, err := s.posts.InsertMany(ctx, parsed.Posts...) if err != nil { _ = s.logs.Delete(ctx, *newLog) return results, err } refreshedLog, err := s.refreshLogCharacters(ctx, *newLog, nil, false) if err != nil { log.Printf("Failed to update characters in newLog %s: %s", newLog.ID, err) } else { newLog = refreshedLog } s.changeService.Submit(ctx, models.ChangeModelLog, "add", true, changekeys.Listed(newLog), newLog) s.changeService.Submit(ctx, models.ChangeModelPost, "add", true, changekeys.Listed(newLog, posts), newLog, posts) results = append(results, newLog) } } default: { return nil, errors.New("Invalid importer: " + importer.String()) } } return results, nil } func (s *LogService) Update(ctx context.Context, id string, update models.LogUpdate) (*models.Log, error) { log, err := s.logs.Find(ctx, id) if err != nil { return nil, err } if err := s.authService.CheckPermission(ctx, "edit", log); err != nil { return nil, err } log, err = s.logs.Update(ctx, *log, update) if err != nil { return nil, err } s.changeService.Submit(ctx, models.ChangeModelLog, "edit", true, changekeys.Listed(log), log) return log, nil } func (s *LogService) SplitLog(ctx context.Context, logId string, startPostId string) (*models.Log, error) { // Find log l, err := s.logs.Find(ctx, logId) if err != nil { return nil, err } if err := s.authService.CheckPermission(ctx, "add", l); err != nil { return nil, err } // Find posts posts, err := s.ListPosts(ctx, &models.PostFilter{LogID: &l.ShortID}) if err != nil { return nil, err } if len(posts) == 0 { return nil, errors.New("cannot split empty log") } // Cut the posts slice. firstPost := posts[0] cutPosts := posts[len(posts):] for i, post := range posts { if post.ID == startPostId { cutPosts = posts[i:] firstPost = post break } } if len(cutPosts) == 0 { return nil, errors.New("post not found") } if len(cutPosts) == len(posts) || firstPost.Time.Equal(l.Date) { return nil, errors.New("cannot move posts") } // Create a new log newLog := &models.Log{ Date: firstPost.Time, ChannelName: l.ChannelName, EventName: l.EventName, } newLog, err = s.logs.Insert(ctx, *newLog) if err != nil { return nil, err } // Put the cut posts in the new log newPosts := make([]*models.Post, 0, len(cutPosts)) for _, post := range cutPosts { postCopy := *post postCopy.ID = "" postCopy.LogID = newLog.ShortID newPost, err := s.posts.Insert(ctx, postCopy) if err != nil { _ = s.logs.Delete(ctx, *newLog) return nil, err } newPosts = append(newPosts, newPost) } // Remove the posts from the old log (this can't error because that'll make a mess) for _, post := range cutPosts { err := s.posts.Delete(ctx, *post) if err != nil { log.Printf("Failed to delete post %s: %s", post.ID, err) } } // Submit the changes s.changeService.Submit(ctx, models.ChangeModelPost, "remove", true, changekeys.Many(l, cutPosts), cutPosts) s.changeService.Submit(ctx, models.ChangeModelLog, "add", true, changekeys.Listed(newLog), newLog) s.changeService.Submit(ctx, models.ChangeModelPost, "add", true, changekeys.Many(newLog, newPosts), newPosts) // Refresh character lists. _, _ = s.refreshLogCharacters(ctx, *l, nil, false) newLog2, err := s.refreshLogCharacters(ctx, *newLog, nil, false) if err == nil { newLog = newLog2 } return newLog, nil } func (s *LogService) MergeLogs(ctx context.Context, targetID string, sourceID string, removeAfter bool) (*models.Log, error) { // Check permissions if err := s.authService.CheckPermission(ctx, "edit", &models.Log{}); err != nil { return nil, err } if removeAfter { if err := s.authService.CheckPermission(ctx, "remove", &models.Log{}); err != nil { return nil, err } } // Merge log posts into log. source, err := s.logs.Find(ctx, sourceID) if err != nil { return nil, errors.New("could not find source log: " + err.Error()) } target, err := s.logs.Find(ctx, targetID) if err != nil { return nil, errors.New("could not find target log: " + err.Error()) } // Get the source posts. posts, err := s.posts.List(ctx, models.PostFilter{LogID: &source.ShortID}) if err != nil { return nil, errors.New("could not fetch source posts: " + err.Error()) } // Associate the posts with the target logs for _, post := range posts { post.ID = "" post.LogID = target.ShortID } // Insert them posts, err = s.posts.InsertMany(ctx, posts...) if err != nil { return nil, errors.New("could not insert posts into target: " + err.Error()) } // Remove other log if removeAfter { err = s.logs.Delete(ctx, *source) if err != nil { return nil, errors.New("posts have been inserted, but could not remove source: " + err.Error()) } s.changeService.Submit(ctx, models.ChangeModelLog, "remove", true, changekeys.Listed(source), source) } // Refresh characters target2, err := s.refreshLogCharacters(ctx, *target, nil, false) if err != nil { log.Printf("Failed to update characters in log %s: %s", target.ID, err) } else { target = target2 } // Submit changes after the target s.changeService.Submit(ctx, models.ChangeModelPost, "add", true, changekeys.Many(target, posts), target, posts) return target, nil } func (s *LogService) AddPost(ctx context.Context, logId string, time time.Time, kind, nick, text string) (*models.Post, error) { if kind == "" || nick == "" || time.IsZero() { return nil, errors.New("kind, nick and time must be non-empty") } l, err := s.logs.Find(ctx, logId) if err != nil { return nil, err } post := &models.Post{ LogID: l.ShortID, Kind: kind, Nick: nick, Text: text, Time: time, } if err := s.authService.CheckPermission(ctx, "add", post); err != nil { return nil, err } post, err = s.posts.Insert(ctx, *post) if err != nil { return nil, err } l2, err := s.refreshLogCharacters(ctx, *l, nil, false) if err != nil { log.Printf("Failed to update characters in log %s: %s", l.ID, err) } else { l = l2 } s.changeService.Submit(ctx, models.ChangeModelPost, "add", true, changekeys.Many(l, post), post) return post, nil } func (s *LogService) EditPost(ctx context.Context, id string, update models.PostUpdate) (*models.Post, error) { if (update.Kind != nil && *update.Kind == "") || (update.Nick != nil && *update.Nick == "") || (update.Text != nil && *update.Text == "") { return nil, errors.New("kind, nick and time must be non-empty") } post, err := s.posts.Find(ctx, id) if err != nil { return nil, err } if err := s.authService.CheckPermission(ctx, "edit", post); err != nil { return nil, err } post, err = s.posts.Update(ctx, *post, update) if err != nil { return nil, err } go func() { l, err := s.logs.Find(context.Background(), post.LogID) if err != nil { return } _, err = s.refreshLogCharacters(ctx, *l, nil, false) if err != nil { log.Printf("Failed to update characters in log %s: %s", l.ID, err) } s.changeService.Submit(ctx, models.ChangeModelPost, "edit", true, changekeys.Many(l, post), post) }() return post, nil } func (s *LogService) MovePost(ctx context.Context, id string, position int) ([]*models.Post, error) { if position < 1 { return nil, repositories.ErrInvalidPosition } post, err := s.posts.Find(ctx, id) if err != nil { return nil, err } if err := s.authService.CheckPermission(ctx, "move", post); err != nil { return nil, err } posts, err := s.posts.Move(ctx, *post, position) if err != nil { return nil, err } go func() { if len(posts) == 0 { return } log, err := s.logs.Find(context.Background(), posts[0].LogID) if err != nil { return } s.changeService.Submit(ctx, models.ChangeModelPost, "move", true, changekeys.Many(log, posts), posts) }() return posts, nil } func (s *LogService) DeletePost(ctx context.Context, id string) (*models.Post, error) { post, err := s.posts.Find(ctx, id) if err != nil { return nil, err } if err := s.authService.CheckPermission(ctx, "remove", post); err != nil { return nil, err } err = s.posts.Delete(ctx, *post) if err != nil { return nil, err } go func() { l, err := s.logs.Find(context.Background(), post.LogID) if err != nil { return } _, err = s.refreshLogCharacters(ctx, *l, nil, false) if err != nil { log.Printf("Failed to update characters in log %s: %s", l.ID, err) } s.changeService.Submit(ctx, models.ChangeModelPost, "remove", true, changekeys.Many(l, post), post) }() return post, nil } func (s *LogService) Delete(ctx context.Context, id string) (*models.Log, error) { log, err := s.logs.Find(ctx, id) if err != nil { return nil, err } if err := s.authService.CheckPermission(ctx, "remove", log); err != nil { return nil, err } err = s.logs.Delete(ctx, *log) if err != nil { return nil, err } s.changeService.Submit(ctx, models.ChangeModelLog, "remove", true, changekeys.Listed(log), log) return log, nil } func (s *LogService) FixImportDateBug(ctx context.Context) error { start := time.Now() logs, err := s.logs.List(ctx, models.LogFilter{}) if err != nil { return err } eg := errgroup.Group{} for i := range logs { l := logs[i] eg.Go(func() error { return s.fixImportDateBug(ctx, *l) }) } err = eg.Wait() if err != nil { return err } log.Printf("Date import bug check finished: logs: %d, duration: %s", len(logs), time.Since(start)) return nil } func (s *LogService) fixImportDateBug(ctx context.Context, l models.Log) error { // Find the log's posts. posts, err := s.ListPosts(ctx, &models.PostFilter{LogID: &l.ShortID}) if err != nil { return err } if len(posts) < 8 { return nil } // Find first action post first := posts[0] fi := 0 for first.Kind != "action" && first.Kind != "text" { fi++ if fi >= len(posts) { return nil } first = posts[fi] } last := posts[len(posts)-1] if first == last { return nil } // Stop here if this log probably isn't affected if last.Time.Sub(first.Time) < time.Hour*72 { return nil } // Find the first post past midnight. midnight := first mi := fi for i, post := range posts[fi+1:] { if post.Time.Hour() < first.Time.Hour() { midnight = post mi = fi + 1 + i break } } if midnight == last { return nil } if len(posts[mi+1:]) == 1 { return nil } hits := 0 prev := midnight for _, offender := range posts[mi+1:] { if offender.Time.Day() != prev.Time.Day() { hits += 1 } prev = offender } if hits < ((len(posts[mi+1:]) * 3) / 4) { return nil } for _, offender := range posts[mi+1:] { ot := offender.Time.UTC() mt := midnight.Time.UTC() y, m, d := mt.Date() hr, mi, se, ns := ot.Hour(), ot.Minute(), ot.Second(), ot.Nanosecond() newTime := time.Date(y, m, d, hr, mi, se, ns, time.UTC) _, err := s.posts.Update(ctx, *offender, models.PostUpdate{Time: &newTime}) if err != nil { return err } } log.Printf("Fixed import date bug in %d posts in log %s", len(posts[mi+1:]), l.ID) return nil } func (s *LogService) RefreshAllLogCharacters(ctx context.Context) error { start := time.Now() // Get all logs logs, err := s.logs.List(ctx, models.LogFilter{}) if err != nil { return err } // Check all characters now instead of later. characters, err := s.characterService.List(ctx, models.CharacterFilter{}) if err != nil { return err } characterMap := s.makeCharacterMap(characters) s.unknownNicksMutex.Lock() for key := range s.unknownNicks { delete(s.unknownNicks, key) } s.unknownNicksMutex.Unlock() tokens := make(chan struct{}, 33) for i := 0; i < 32; i++ { tokens <- struct{}{} } eg := errgroup.Group{} for i := range logs { l := logs[i] eg.Go(func() error { <-tokens defer func() { tokens <- struct{}{} }() _, err := s.refreshLogCharacters(ctx, *l, characterMap, true) return err }) } err = eg.Wait() if err != nil { return err } s.unknownNicksMutex.Lock() unknownCount := len(s.unknownNicks) s.unknownNicksMutex.Unlock() log.Printf("Full log character refresh complete; nicks: %d, unknowns: %d, logs: %d, duration: %s", len(characterMap), unknownCount, len(logs), time.Since(start)) return nil } func (s *LogService) RefreshLogCharacters(ctx context.Context, log models.Log) (*models.Log, error) { return s.refreshLogCharacters(ctx, log, nil, false) } func (s *LogService) refreshLogCharacters(ctx context.Context, log models.Log, characterMap map[string]*models.Character, useUnknownNicks bool) (*models.Log, error) { posts, err := s.ListPosts(ctx, &models.PostFilter{LogID: &log.ShortID}) if err != nil { return nil, err } counts := make(map[string]int) added := make(map[string]bool) removed := make(map[string]bool) for _, post := range posts { if post.Kind == "text" || post.Kind == "action" { if strings.HasPrefix(post.Text, "(") || strings.Contains(post.Nick, "(") || strings.Contains(post.Nick, "[E]") || strings.HasSuffix(post.Nick, "|") { continue } // Clean up the nick (remove possessive suffix, comma, formatting stuff) if strings.HasSuffix(post.Nick, "'s") || strings.HasSuffix(post.Nick, "`s") { post.Nick = post.Nick[:len(post.Nick)-2] } else if strings.HasSuffix(post.Nick, "'") || strings.HasSuffix(post.Nick, "`") || strings.HasSuffix(post.Nick, ",") || strings.HasSuffix(post.Nick, "\x0f") { post.Nick = post.Nick[:len(post.Nick)-1] } added[post.Nick] = true counts[post.Nick]++ } if post.Kind == "chars" { tokens := strings.Fields(post.Text) for _, token := range tokens { if strings.HasPrefix(token, "-") { removed[token[1:]] = true } else { added[strings.Replace(token, "+", "", 1)] = true } } } } nicks := make([]string, 0, len(added)) for nick := range added { if added[nick] && !removed[nick] { nicks = append(nicks, nick) } } if characterMap == nil { characters, err := s.characterService.List(ctx, models.CharacterFilter{Nicks: nicks}) if err != nil { return nil, err } characterMap = s.makeCharacterMap(characters) } log.CharacterIDs = log.CharacterIDs[:0] for key := range added { delete(added, key) } unknowned := make(map[string]bool) for _, nick := range nicks { character := characterMap[nick] if character == nil { if useUnknownNicks && !unknowned[nick] { unknowned[nick] = true s.unknownNicksMutex.Lock() s.unknownNicks[nick]++ s.unknownNicksMutex.Unlock() } continue } else if added[character.ID] { continue } added[character.ID] = true log.CharacterIDs = append(log.CharacterIDs, character.ID) } return s.logs.Update(ctx, log, models.LogUpdate{CharacterIDs: log.CharacterIDs}) } func (s *LogService) makeCharacterMap(characters []*models.Character) map[string]*models.Character { characterMap := make(map[string]*models.Character, len(characters)*3) for _, character := range characters { for _, nick := range character.Nicks { characterMap[nick] = character } } return characterMap } func (s *LogService) NextLogs(ctx context.Context, log *models.Log) ([]*models.LogSuggestion, error) { minDate := log.Date.Add(time.Millisecond) logs, err := s.logs.List(ctx, models.LogFilter{ MinDate: &minDate, }) if err != nil { return nil, err } sort.Slice(logs, func(i, j int) bool { return logs[i].Date.Before(logs[j].Date) }) return s.findSuggestions(ctx, log, logs) } func (s *LogService) PrevLogs(ctx context.Context, log *models.Log) ([]*models.LogSuggestion, error) { logs, err := s.logs.List(ctx, models.LogFilter{ MaxDate: &log.Date, }) if err != nil { return nil, err } return s.findSuggestions(ctx, log, logs) } func (s *LogService) findSuggestions(ctx context.Context, log *models.Log, logs []*models.Log) ([]*models.LogSuggestion, error) { characters, err := s.characterService.List(ctx, models.CharacterFilter{ IDs: log.CharacterIDs, }) if err != nil { return nil, err } charIntersect := func(l1, l2 *models.Log) []*models.Character { results := make([]*models.Character, 0, len(l1.CharacterIDs)) for _, c1 := range characters { for _, c2ID := range l2.CharacterIDs { if c1.ID == c2ID { results = append(results, c1) break } } } return results } groupKey := func(characters []*models.Character) string { if len(characters) == 0 { return "" } else if len(characters) == 1 { return characters[0].ID } builder := strings.Builder{} builder.WriteString(characters[0].ID) for _, character := range characters { builder.WriteRune(',') builder.WriteString(character.ID) } return builder.String() } suggestions := make([]*models.LogSuggestion, 0, 16) foundGroups := make(map[string]bool) foundChannel := false for _, log2 := range logs { hasEvent := log.EventName != "" && log2.EventName == log.EventName hasChannel := log.ChannelName == log2.ChannelName characters := charIntersect(log, log2) groupKey := groupKey(characters) suggestion := &models.LogSuggestion{ Log: log2, Characters: characters, HasChannel: hasChannel, HasEvent: hasEvent, } if hasChannel && foundChannel { foundChannel = true foundGroups[groupKey] = true suggestions = append(suggestions, suggestion) } else if hasEvent { foundGroups[groupKey] = true suggestions = append(suggestions, suggestion) } else if len(suggestions) < 8 && !hasEvent && len(characters) > 1 && !foundGroups[groupKey] { foundGroups[groupKey] = true suggestions = append(suggestions, suggestion) } } return suggestions, nil }