8 changed files with 377 additions and 5 deletions
-
7graph2/resolvers/log.go
-
8graph2/schema/types/Log.gql
-
4models/log-importer.go
-
40services/logs.go
-
161services/parsers/irccloud.go
-
154services/parsers/irccloud_test.go
-
4services/parsers/mirclike.go
-
4services/parsers/mirclike_test.go
@ -0,0 +1,161 @@ |
|||
package parsers |
|||
|
|||
import ( |
|||
"errors" |
|||
"fmt" |
|||
"git.aiterp.net/rpdata/api/models" |
|||
"strings" |
|||
"time" |
|||
) |
|||
|
|||
var ErrSkip = errors.New("parsers: skip this post") |
|||
|
|||
func IRCCloudLogs(data string, location *time.Location, threshold time.Duration) ([]ParsedLog, error) { |
|||
lines := strings.Split(data, "\n") |
|||
pos := 0 |
|||
results := make([]ParsedLog, 0, 8) |
|||
for pos < len(lines) { |
|||
log, n, err := IRCCloudLog(lines[pos:], location, threshold) |
|||
if err != nil { |
|||
if err == ErrEmptyLog { |
|||
pos += n |
|||
continue |
|||
} |
|||
|
|||
return nil, err |
|||
} |
|||
|
|||
pos += n |
|||
results = append(results, *log) |
|||
} |
|||
|
|||
return results, nil |
|||
} |
|||
|
|||
// IRCCloudLog parses the log and returns the things that can be gleamed from them.
|
|||
func IRCCloudLog(lines []string, location *time.Location, threshold time.Duration) (*ParsedLog, int, error) { |
|||
posts := make([]*models.Post, 0, len(lines)) |
|||
prev := (*models.Post)(nil) |
|||
amount := 0 |
|||
|
|||
for _, line := range lines { |
|||
line = strings.Trim(line, "\r\t ") |
|||
if len(line) < 1 { |
|||
amount += 1 |
|||
continue |
|||
} |
|||
|
|||
post, err := IRCCloudPost(line, location) |
|||
if err == ErrSkip { |
|||
amount += 1 |
|||
continue |
|||
} else if err != nil { |
|||
return nil, -1, err |
|||
} |
|||
|
|||
if prev != nil { |
|||
if post.Time.Sub(prev.Time) >= threshold { |
|||
break |
|||
} |
|||
|
|||
post.Position = prev.Position + 1 |
|||
} else { |
|||
post.Position = 1 |
|||
} |
|||
|
|||
posts = append(posts, &post) |
|||
prev = &post |
|||
amount += 1 |
|||
} |
|||
|
|||
if len(posts) == 0 { |
|||
return nil, amount, ErrEmptyLog |
|||
} |
|||
|
|||
log := models.Log{ |
|||
Date: posts[0].Time, |
|||
} |
|||
|
|||
return &ParsedLog{log, posts}, amount, nil |
|||
} |
|||
|
|||
// IRCCloudPost parses a post from a mirc-like line. If the previous post is included (it can be empty), it will be used
|
|||
// to determine whether midnight has passed.
|
|||
func IRCCloudPost(line string, tz *time.Location) (models.Post, error) { |
|||
// Do basic validation
|
|||
line = strings.Trim(line, " \t\n\r") |
|||
if len(line) == 0 || !strings.HasPrefix(line, "[") { |
|||
return models.Post{}, &ParseError{ |
|||
Line: line, |
|||
Problem: "no timestamp", |
|||
} |
|||
} |
|||
|
|||
// Parse timestamp
|
|||
tsEndIndex := strings.IndexByte(line, ']') |
|||
if tsEndIndex == -1 || len(line) < tsEndIndex+5 { |
|||
return models.Post{}, &ParseError{ |
|||
Line: line, |
|||
Problem: "incomplete timestamp", |
|||
} |
|||
} |
|||
tsStr := line[1:tsEndIndex] |
|||
|
|||
ts, err := time.ParseInLocation("2006-01-02 15:04:05", tsStr, tz) |
|||
if err != nil { |
|||
return models.Post{}, &ParseError{ |
|||
Line: line, |
|||
Problem: fmt.Sprintf("Could not parse date: %s", err.Error()), |
|||
} |
|||
} |
|||
|
|||
if strings.HasPrefix(line[tsEndIndex+2:], "—") { |
|||
split := strings.SplitN(line[tsEndIndex+6:], " ", 2) |
|||
if len(split) == 1 { |
|||
return models.Post{}, &ParseError{ |
|||
Line: line, |
|||
Problem: "post is empty", |
|||
} |
|||
} |
|||
|
|||
post := models.Post{ |
|||
ID: "UNASSIGNED", |
|||
LogID: "UNASSIGNED", |
|||
Time: ts, |
|||
Kind: "action", |
|||
Nick: strings.Trim(strings.TrimLeft(split[0], "+@!~"), "\u001F"), |
|||
Text: split[1], |
|||
} |
|||
|
|||
if strings.HasPrefix(post.Nick, "=") { |
|||
post.Kind = "scene" |
|||
} |
|||
|
|||
return post, nil |
|||
} else if line[tsEndIndex+2] == '<' { |
|||
split := strings.SplitN(line[tsEndIndex+2:], " ", 2) |
|||
if len(split) == 1 { |
|||
return models.Post{}, &ParseError{ |
|||
Line: line, |
|||
Problem: "post is empty", |
|||
} |
|||
} |
|||
|
|||
post := models.Post{ |
|||
ID: "UNASSIGNED", |
|||
LogID: "UNASSIGNED", |
|||
Time: ts, |
|||
Kind: "text", |
|||
Nick: strings.Trim(strings.TrimLeft(split[0][1:len(split[0])-1], "+@!~"), "\u001F"), |
|||
Text: split[1], |
|||
} |
|||
|
|||
if strings.HasPrefix(post.Nick, "=") { |
|||
post.Kind = "scene" |
|||
} |
|||
|
|||
return post, nil |
|||
} else { |
|||
return models.Post{}, ErrSkip |
|||
} |
|||
} |
|||
@ -0,0 +1,154 @@ |
|||
package parsers |
|||
|
|||
import ( |
|||
"fmt" |
|||
"github.com/stretchr/testify/assert" |
|||
"testing" |
|||
"time" |
|||
) |
|||
|
|||
func TestIRCCloudPost(t *testing.T) { |
|||
table := []struct { |
|||
Input string |
|||
TS string |
|||
Kind string |
|||
Nick string |
|||
Text string |
|||
Skipped bool |
|||
}{ |
|||
{ |
|||
"[2016-07-18 12:57:17] — \u001FCharacter_Name\u001F does things.", |
|||
"2016-07-18 12:57:17", "action", "Character_Name", "does things.", |
|||
false, |
|||
}, |
|||
{ |
|||
"[2016-07-18 13:04:06] — Character keeps their mouth shut again for the time beings.", |
|||
"2016-07-18 13:04:06", "action", "Character", "keeps their mouth shut again for the time beings.", |
|||
false, |
|||
}, |
|||
{ |
|||
"[2016-07-18 12:34:42] <Victoria_Steels> \"I can keep it professional.\"", |
|||
"2016-07-18 12:34:42", "text", "Victoria_Steels", "\"I can keep it professional.\"", |
|||
false, |
|||
}, |
|||
{ |
|||
"[2016-07-18 10:41:10] * UserZzz → User", |
|||
"", "", "", "", |
|||
true, |
|||
}, |
|||
{ |
|||
"[2016-07-18 12:53:55] → Someone joined", |
|||
"", "", "", "", |
|||
true, |
|||
}, |
|||
{ |
|||
"[2016-07-18 22:14:15] ⇐ SomeoneElse quit (s@example.com): Ping timeout: 547 seconds", |
|||
"", "", "", "", |
|||
true, |
|||
}, |
|||
} |
|||
|
|||
for i, row := range table { |
|||
t.Run(fmt.Sprintf("Row_%d", i), func(t *testing.T) { |
|||
post, err := IRCCloudPost(row.Input, time.UTC) |
|||
if err != nil && err != ErrSkip { |
|||
t.Fatal("Could not parse post:", err) |
|||
} |
|||
|
|||
if row.Skipped { |
|||
if err == ErrSkip { |
|||
return |
|||
} |
|||
|
|||
t.Fatal("Row should be skipped") |
|||
} |
|||
if !row.Skipped && err == ErrSkip { |
|||
t.Fatal("Row should not be skipped, but is") |
|||
} |
|||
|
|||
assert.Equal(t, row.TS, post.Time.Format("2006-01-02 15:04:05"), "Timestamps should match.") |
|||
assert.Equal(t, row.Kind, post.Kind, "Kinds should match.") |
|||
assert.Equal(t, row.Nick, post.Nick, "Kinds should match.") |
|||
assert.Equal(t, row.Text, post.Text, "Kinds should match.") |
|||
}) |
|||
} |
|||
} |
|||
|
|||
func TestIRCCloudLog(t *testing.T) { |
|||
type logLine struct { |
|||
TS string |
|||
Kind string |
|||
Nick string |
|||
Text string |
|||
} |
|||
|
|||
logs := [][]logLine{ |
|||
{ |
|||
{"2011-09-23 15:37:16", "action", "Jason_Wolfe", "sheds a bit of the tension he was harboring as Markus extends his hand."}, |
|||
{"2011-09-23 15:41:15", "action", "Markus_Vasquez", "gives another nod, but there's no smile from him."}, |
|||
{"2011-09-23 15:47:22", "action", "Jason_Wolfe", "fights the temptation to shove his hand back in his pocket as he notices Markus."}, |
|||
{"2011-09-23 15:47:22", "action", "Jason_Wolfe", "warmer to wear."}, |
|||
{"2011-09-23 15:47:32", "text", "Dante", "((you're*))"}, |
|||
{"2011-09-23 15:48:45", "action", "Markus_Vasquez", "grunts as they walk toward the door, shaking his head slightly."}, |
|||
}, |
|||
{ |
|||
{"2011-09-25 21:03:55", "scene", "=Scene=", "It's early evening on the 11th of November, and the weather is clear."}, |
|||
{"2011-09-25 21:03:55", "scene", "=Scene=", "café on the eastern edge of town."}, |
|||
{"2011-09-25 21:08:33", "action", "Sofia_Tennhausen", "is sat on a small table outside the cafe, a cup of lukewarm coffee in her hand."}, |
|||
}, |
|||
} |
|||
|
|||
results, err := IRCCloudLogs(testLog, time.UTC, time.Hour*6) |
|||
if err != nil { |
|||
t.Fatal("Parse", err) |
|||
} |
|||
|
|||
if len(results) != len(logs) { |
|||
t.Fatal("Two logs expected, got", len(logs)) |
|||
} |
|||
|
|||
for i, log := range logs { |
|||
result := results[i] |
|||
|
|||
if len(result.Posts) != len(log) { |
|||
t.Error(len(log), "posts expected in log", i, "but got", len(result.Posts)) |
|||
for _, post := range result.Posts { |
|||
t.Log(*post) |
|||
} |
|||
|
|||
return |
|||
} |
|||
|
|||
for j, post := range result.Posts { |
|||
t.Run(fmt.Sprintf("Log_%d_Post_%d", i, j), func(t *testing.T) { |
|||
assert.Equal(t, log[j].TS, post.Time.Format("2006-01-02 15:04:05"), "Timestamps should match.") |
|||
assert.Equal(t, log[j].Kind, post.Kind, "Kinds should match.") |
|||
assert.Equal(t, log[j].Nick, post.Nick, "Kinds should match.") |
|||
assert.Equal(t, log[j].Text, post.Text, "Kinds should match.") |
|||
}) |
|||
} |
|||
|
|||
} |
|||
} |
|||
|
|||
var testLog = ` |
|||
[2011-09-23 15:37:16] — Jason_Wolfe sheds a bit of the tension he was harboring as Markus extends his hand. |
|||
[2011-09-23 15:41:15] — Markus_Vasquez gives another nod, but there's no smile from him. |
|||
|
|||
|
|||
[2011-09-23 15:47:22] — Jason_Wolfe fights the temptation to shove his hand back in his pocket as he notices Markus. |
|||
[2011-09-23 15:47:22] — Jason_Wolfe warmer to wear. |
|||
[2011-09-23 15:47:32] <Dante> ((you're*)) |
|||
[2011-09-23 15:48:45] — Markus_Vasquez grunts as they walk toward the door, shaking his head slightly. |
|||
[2011-09-23 18:00:28] ⇐ Tyranniac quit (Tyranniac@example.com): Ping timeout: 260 seconds |
|||
[2011-09-23 18:58:39] → Tyranniac joined (Tyranniac@example.com) |
|||
[2011-09-23 18:58:39] * ChanServ set +v Tyranniac |
|||
[2011-09-24 01:45:21] → Hobo joined (Hobo@example.com) |
|||
[2011-09-24 11:37:07] ⇐ Hobo quit (Hobo@example.com): |
|||
[2011-09-24 21:01:34] ⇐ Tyranniac quit (Tyranniac@example.com): Ping timeout: 264 seconds |
|||
[2011-09-24 23:00:32] → Tyranniac joined (Tyranniac@example.com) |
|||
[2011-09-25 20:54:43] * Bowe → Sofia_Tennhausen |
|||
[2011-09-25 21:03:55] <=Scene=> It's early evening on the 11th of November, and the weather is clear. |
|||
[2011-09-25 21:03:55] <=Scene=> café on the eastern edge of town. |
|||
[2011-09-25 21:08:33] — Sofia_Tennhausen is sat on a small table outside the cafe, a cup of lukewarm coffee in her hand. |
|||
` |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue