Browse Source

logs: Added mirclike parsers.

module-madness-pointers
Gisle Aune 6 years ago
parent
commit
caa7727c02
  1. 42
      Gopkg.lock
  2. 5
      Gopkg.toml
  3. 48
      internal/importers/mirclike/log.go
  4. 69
      internal/importers/mirclike/log_test.go
  5. 92
      internal/importers/mirclike/post.go
  6. 126
      internal/importers/mirclike/post_test.go

42
Gopkg.lock

@ -23,6 +23,12 @@
revision = "3d21ba515fe27b856f230847e856431ae1724adc" revision = "3d21ba515fe27b856f230847e856431ae1724adc"
version = "v1.0.0" version = "v1.0.0"
[[projects]]
name = "github.com/davecgh/go-spew"
packages = ["spew"]
revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73"
version = "v1.1.1"
[[projects]] [[projects]]
name = "github.com/dgrijalva/jwt-go" name = "github.com/dgrijalva/jwt-go"
packages = ["."] packages = ["."]
@ -71,27 +77,6 @@
revision = "78139374585c29dcb97b8f33089ed11959e4be59" revision = "78139374585c29dcb97b8f33089ed11959e4be59"
version = "v5" version = "v5"
[[projects]]
branch = "master"
name = "github.com/graph-gophers/graphql-go"
packages = [
".",
"errors",
"internal/common",
"internal/exec",
"internal/exec/packer",
"internal/exec/resolvable",
"internal/exec/selected",
"internal/query",
"internal/schema",
"internal/validation",
"introspection",
"log",
"relay",
"trace"
]
revision = "9ebf33af539ab8cb832c7107bc0a978ca8dbc0de"
[[projects]] [[projects]]
name = "github.com/hashicorp/golang-lru" name = "github.com/hashicorp/golang-lru"
packages = [ packages = [
@ -133,7 +118,6 @@
name = "github.com/opentracing/opentracing-go" name = "github.com/opentracing/opentracing-go"
packages = [ packages = [
".", ".",
"ext",
"log" "log"
] ]
revision = "1949ddbfd147afd4d964a9f00b24eb291e0e7c38" revision = "1949ddbfd147afd4d964a9f00b24eb291e0e7c38"
@ -145,12 +129,24 @@
revision = "645ef00459ed84a119197bfb8d8205042c6df63d" revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
version = "v0.8.0" version = "v0.8.0"
[[projects]]
name = "github.com/pmezard/go-difflib"
packages = ["difflib"]
revision = "792786c7400a136282c1664665ae0a8db921c6c2"
version = "v1.0.0"
[[projects]] [[projects]]
name = "github.com/sirupsen/logrus" name = "github.com/sirupsen/logrus"
packages = ["."] packages = ["."]
revision = "c155da19408a8799da419ed3eeb0cb5db0ad5dbc" revision = "c155da19408a8799da419ed3eeb0cb5db0ad5dbc"
version = "v1.0.5" version = "v1.0.5"
[[projects]]
name = "github.com/stretchr/testify"
packages = ["assert"]
revision = "f35b8ab0b5a2cef36673838d662e249dd9c94686"
version = "v1.2.2"
[[projects]] [[projects]]
name = "github.com/urfave/cli" name = "github.com/urfave/cli"
packages = ["."] packages = ["."]
@ -249,6 +245,6 @@
[solve-meta] [solve-meta]
analyzer-name = "dep" analyzer-name = "dep"
analyzer-version = 1 analyzer-version = 1
inputs-digest = "3c9430afa8b9260026925c8f496af934e2e573a2bd4587bb634fd12a20ce6c2c"
inputs-digest = "eb8fc099a909f5d8f756b24fbcb6cc2856e132bd3ed16b8fe512f22fabb99596"
solver-name = "gps-cdcl" solver-name = "gps-cdcl"
solver-version = 1 solver-version = 1

5
Gopkg.toml

@ -40,4 +40,7 @@ required = [ "github.com/99designs/gqlgen" ]
[[constraint]] [[constraint]]
branch = "master" branch = "master"
name = "github.com/graph-gophers/graphql-go"
name = "github.com/graph-gophers/graphql-go"
[[constraint]]
name = "github.com/stretchr/testify"
version = "1.2.2"

48
internal/importers/mirclike/log.go

@ -0,0 +1,48 @@
package mirclike
import (
"errors"
"strings"
"time"
"git.aiterp.net/rpdata/api/models"
)
// ErrEmptyLog is returned by ParseLog if there are no (valid) posts in the log.
var ErrEmptyLog = errors.New("No valid posts found in log")
// ParseLog parses the log and returns the things that can be gleamed from them.
func ParseLog(data string, date time.Time, strict bool) (models.Log, []models.Post, error) {
lines := strings.Split(data, "\n")
posts := make([]models.Post, 0, len(lines))
prev := models.Post{}
for _, line := range lines {
line = strings.Trim(line, "\r\t  ")
if len(line) < 1 {
continue
}
post, err := ParsePost(line, date, prev)
if err != nil {
if strict {
return models.Log{}, nil, err
}
continue
}
posts = append(posts, post)
prev = post
}
if len(posts) == 0 {
return models.Log{}, nil, ErrEmptyLog
}
log := models.Log{
Date: posts[0].Time,
}
return log, posts, nil
}

69
internal/importers/mirclike/log_test.go

@ -0,0 +1,69 @@
package mirclike_test
import (
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
"git.aiterp.net/rpdata/api/internal/importers/mirclike"
"git.aiterp.net/rpdata/api/models"
)
var testLog = strings.Join([]string{
"[11:21] * Va`ynna_Atana returns to the apartment at the end of another long day. She hangs up her jacket and a green scarf next to the door before going straight to the bathroom; she leaves the door open, though. She walked home with Uvena this time, but the two parted ways downstairs.",
"",
"[11:21] <=Scene=> The weather outside is not pleasant in the slightest. The storm is still going on, visibility is reduced and the worst gusts of wind can be felt inside the apartment as a faint shudder. ",
"",
"[11:",
" [11:27] <Test> Stuff and things.",
}, "\r\n")
var testLogPosts = []models.Post{
{
ID: "UNASSIGNED",
LogID: "UNASSIGNED",
Time: parseDate(nil, "2018-05-11 11:21:00"),
Kind: "action",
Nick: "Va`ynna_Atana",
Position: 1,
Text: "returns to the apartment at the end of another long day. She hangs up her jacket and a green scarf next to the door before going straight to the bathroom; she leaves the door open, though. She walked home with Uvena this time, but the two parted ways downstairs.",
},
{
ID: "UNASSIGNED",
LogID: "UNASSIGNED",
Time: parseDate(nil, "2018-05-11 11:21:00"),
Kind: "scene",
Nick: "=Scene=",
Position: 2,
Text: "The weather outside is not pleasant in the slightest. The storm is still going on, visibility is reduced and the worst gusts of wind can be felt inside the apartment as a faint shudder.",
},
{
ID: "UNASSIGNED",
LogID: "UNASSIGNED",
Time: parseDate(nil, "2018-05-11 11:27:00"),
Kind: "text",
Nick: "Test",
Position: 3,
Text: "Stuff and things.",
},
}
func TestParseLog(t *testing.T) {
log, posts, err := mirclike.ParseLog(testLog, parseDate(t, "2018-05-11 00:00:00"), false)
if err != nil {
t.Fatal("ParseLog", err)
}
assert.Equal(t, testLogPosts, posts)
assert.Equal(t, posts[0].Time, log.Date, "Log's date should be the first post's.")
}
func TestParseLogErrors(t *testing.T) {
_, _, err1 := mirclike.ParseLog("\n\n\n\n\n \n\t\r\n", time.Time{}, false)
_, _, err2 := mirclike.ParseLog("\n\n\n\n\n[14:57]* Stuff \n\t\r\n", time.Time{}, true)
assert.Equal(t, mirclike.ErrEmptyLog, err1)
assert.Equal(t, mirclike.ErrNotPost, err2)
}

92
internal/importers/mirclike/post.go

@ -0,0 +1,92 @@
package mirclike
import (
"errors"
"strconv"
"strings"
"time"
"git.aiterp.net/rpdata/api/models"
)
// ErrNotPost is returned by parsePost if the line is empty or not a post.
var ErrNotPost = errors.New("not a post")
// ParsePost parses a post from a mirc-like line. If the previous post is included (it can be empty), it will be used
// to determine whether midnight has passed.
func ParsePost(line string, date time.Time, prev models.Post) (models.Post, error) {
// Do basic validation
line = strings.Trim(line, "  \t\n\r")
if len(line) == 0 || !strings.HasPrefix(line, "[") {
return models.Post{}, ErrNotPost
}
// Parse timestamp
tsEndIndex := strings.IndexByte(line, ']')
if tsEndIndex == -1 || len(line) < tsEndIndex+5 {
return models.Post{}, ErrNotPost
}
tsStr := line[1:tsEndIndex]
tsSplit := strings.Split(tsStr, ":")
tsUnits := make([]int, len(tsSplit))
if len(tsSplit) < 2 {
return models.Post{}, ErrNotPost
}
for i := range tsSplit {
n, err := strconv.Atoi(tsSplit[i])
if err != nil {
return models.Post{}, ErrNotPost
}
tsUnits[i] = n
}
if len(tsUnits) == 2 {
tsUnits = append(tsUnits, 0)
}
// Determine timestamp from parsed data and previous post.
ts := time.Date(date.Year(), date.Month(), date.Day(), tsUnits[0], tsUnits[1], tsUnits[2], 0, date.Location())
if !prev.Time.IsZero() && prev.Time.Sub(ts) > 30*time.Minute {
ts = time.Date(prev.Time.Year(), prev.Time.Month(), prev.Time.Day()+1, tsUnits[0], tsUnits[1], tsUnits[2], 0, date.Location())
}
if line[tsEndIndex+2] == '*' {
split := strings.SplitN(line[tsEndIndex+4:], " ", 2)
post := models.Post{
ID: "UNASSIGNED",
LogID: "UNASSIGNED",
Time: ts,
Kind: "action",
Nick: strings.TrimLeft(split[0], "+@!~"),
Text: split[1],
Position: prev.Position + 1,
}
if post.Nick[0] == '=' {
post.Kind = "scene"
}
return post, nil
} else if line[tsEndIndex+2] == '<' {
split := strings.SplitN(line[tsEndIndex+2:], " ", 2)
post := models.Post{
ID: "UNASSIGNED",
LogID: "UNASSIGNED",
Time: ts,
Kind: "text",
Nick: strings.TrimLeft(split[0][1:len(split[0])-1], "+@!~"),
Text: split[1],
Position: prev.Position + 1,
}
if post.Nick[0] == '=' {
post.Kind = "scene"
}
return post, nil
} else {
return models.Post{}, ErrNotPost
}
}

126
internal/importers/mirclike/post_test.go

@ -0,0 +1,126 @@
package mirclike_test
import (
"fmt"
"testing"
"time"
"github.com/stretchr/testify/assert"
"git.aiterp.net/rpdata/api/internal/importers/mirclike"
"git.aiterp.net/rpdata/api/models"
)
func TestParsePost(t *testing.T) {
table := []struct {
Input string
TS string
Kind string
Nick string
Text string
}{
{
"[12:34] * Stuff does things.",
"12:34:00", "action", "Stuff", "does things.",
},
{
"[12:34] <Stuff> Things said.",
"12:34:00", "text", "Stuff", "Things said.",
},
{
"[13:36:59] <Stuff> Things said.",
"13:36:59", "text", "Stuff", "Things said.",
},
{
"[23:59] <=Scene=> Scenery and such.",
"23:59:00", "scene", "=Scene=", "Scenery and such.",
},
{
"[01:10:11] * =Scene= Scenery and such from the forum or mIRC using my old script.",
"01:10:11", "scene", "=Scene=", "Scenery and such from the forum or mIRC using my old script.",
},
}
for i, row := range table {
t.Run(fmt.Sprintf("Row_%d", i), func(t *testing.T) {
post, err := mirclike.ParsePost(row.Input, time.Now(), models.Post{})
if err != nil {
t.Fatal("Could not parse post:", err)
}
assert.Equal(t, row.TS, post.Time.Format("15:04:05"), "Timestamps should match.")
assert.Equal(t, row.Kind, post.Kind, "Kinds should match.")
assert.Equal(t, row.Nick, post.Nick, "Kinds should match.")
assert.Equal(t, row.Text, post.Text, "Kinds should match.")
})
}
}
func TestParsePostErrors(t *testing.T) {
table := []struct {
Input string
Err error
}{
{"[12:34] <Stuff> Things said.", nil},
{"[12:34] >Stuff> Things said.", mirclike.ErrNotPost},
{"12:34] <Stuff> Things said.", mirclike.ErrNotPost},
{"* Stuff Things said.", mirclike.ErrNotPost},
{"", mirclike.ErrNotPost},
{"[12:34 <Stuff> Things said.", mirclike.ErrNotPost},
{"[TE:XT] <Stuff> Things said.", mirclike.ErrNotPost},
{"[10] <Stuff> Things said.", mirclike.ErrNotPost},
{"[12:34:56:789] <Stuff> Things said.", nil},
{"[12:34:56.789] <Stuff> Things said.", mirclike.ErrNotPost},
}
for i, row := range table {
t.Run(fmt.Sprintf("Row_%d", i), func(t *testing.T) {
_, err := mirclike.ParsePost(row.Input, time.Now(), models.Post{})
assert.Equal(t, row.Err, err, "Error should match")
})
}
}
func TestParseNextDay(t *testing.T) {
table := []struct {
Prev time.Time
TS string
Time time.Time
}{
{Prev: parseDate(t, "2019-01-12 12:00:00"), TS: "12:01", Time: parseDate(t, "2019-01-12 12:01:00")},
{Prev: parseDate(t, "2019-01-12 12:00:00"), TS: "11:53:13", Time: parseDate(t, "2019-01-12 11:53:13")},
{Prev: parseDate(t, "2019-04-08 23:51:59"), TS: "00:09", Time: parseDate(t, "2019-04-09 00:09:00")},
{Prev: parseDate(t, "2019-01-12 12:00:00"), TS: "11:29:59", Time: parseDate(t, "2019-01-13 11:29:59")},
}
for i, row := range table {
t.Run(fmt.Sprintf("Row_%d", i), func(t *testing.T) {
input := fmt.Sprintf("[%s] * Stuff does things.", row.TS)
post, err := mirclike.ParsePost(input, row.Prev, models.Post{Time: row.Prev})
if err != nil {
t.Fatal("Could not parse post:", err)
}
assert.Equal(t, row.Time, post.Time)
})
}
}
func parseDate(t *testing.T, date string) time.Time {
result, err := time.Parse("2006-01-02 15:04:05", date)
if err != nil {
if t != nil {
t.Fatal("Could not parse date", date, err)
} else {
panic("Could not parse date: " + err.Error())
}
}
return result
}
func formatDate(date time.Time) string {
return date.UTC().Format("2006-01-02 15:04:05")
}
Loading…
Cancel
Save