Logs and posts are now on the new architecture.
the build was successful

This commit is contained in:
2019-06-18 20:34:58 +02:00
parent e194b6536e
commit 73a1c79903
37 changed files with 1575 additions and 922 deletions
+54
View File
@@ -0,0 +1,54 @@
package generate
import (
"crypto/rand"
"encoding/binary"
mrand "math/rand"
"strconv"
"strings"
)
// ID generates an ID using crypto-random, falling back to math random when that fails
// to avoid disrupting operation because of a faulty RNG.
func ID(prefix string, length int) string {
var data [32]byte
result := strings.Builder{}
result.Grow(length + 32)
result.WriteString(prefix)
pos := 0
for result.Len() < length {
if pos == 0 {
randRead(data[:])
}
result.WriteString(strconv.FormatUint(binary.BigEndian.Uint64(data[pos:pos+8]), 36))
pos = (pos + 8) % 32
}
return result.String()[:length]
}
func randRead(data []byte) {
n, err := rand.Read(data)
if err != nil {
mrand.Read(data[n:])
}
}
// PostID generates a post ID.
func PostID() string {
return ID("P", 16)
}
// StoryID generates a post ID.
func StoryID() string {
return ID("S", 16)
}
// ChapterID generates a post ID.
func ChapterID() string {
return ID("SC", 24)
}
+20
View File
@@ -0,0 +1,20 @@
package generate
import (
"fmt"
"git.aiterp.net/rpdata/api/models"
"time"
)
// LogID generates a log ID in the format is 2019-05-22_191341547_RedrockAgency
func LogID(log models.Log) string {
if len(log.ChannelName) < 1 {
panic("ChannelName is not valid (validate input before calling this function!)")
}
datetime := log.Date.UTC().Format("2006-01-02_150405")
milliseconds := (log.Date.UnixNano() % int64(time.Second)) / 1000000
channelName := log.ChannelName[1:]
return fmt.Sprintf("%s%03d_%s", datetime, milliseconds, channelName)
}
-101
View File
@@ -1,101 +0,0 @@
package forumlog
import (
"bufio"
"fmt"
"strings"
"time"
"git.aiterp.net/rpdata/api/internal/importers/mirclike"
"git.aiterp.net/rpdata/api/models"
)
// A ParsedLog contains the parsed log header and its posts.
type ParsedLog struct {
Log models.Log
Posts []models.Post
}
// ParseLogs parses the logs from the data.
func ParseLogs(data string, tz *time.Location) ([]ParsedLog, error) {
metadata := ParseMetadata(data)
results := make([]ParsedLog, 0, len(metadata["Date"]))
scanner := bufio.NewScanner(strings.NewReader(data))
for i, dateStr := range metadata["Date"] {
// Parse date
date, err := time.ParseInLocation("January 2, 2006", dateStr, tz)
if err != nil {
return nil, fmt.Errorf("Failed to parse date #%d: %#+v is not the in the correct format of \"January 2, 2006\"", i+1, dateStr)
}
// Parse posts
posts := make([]models.Post, 0, 128)
parsing := false
prev := ""
prevPost := models.Post{}
for scanner.Scan() {
line := strings.Trim(scanner.Text(), "\r\t  ")
// Skip lines with links to other logs using the --> and <-- notation.
if strings.HasPrefix(line, "-") || strings.HasPrefix(line, "<") {
prev = line
continue
}
// If parsing and reaching a double empty-line, the session is done.
if parsing && len(line) < 2 && len(prev) < 2 {
break
}
// Skip empty lines, but record them as thep revious for the above check.
if len(line) < 2 {
prev = line
continue
}
// If not parsing, skip until the first mirclike post.
if !parsing {
if strings.HasPrefix(line, "[") {
parsing = true
} else {
prev = line
continue
}
}
// Parse the post.
post, err := mirclike.ParsePost(line, date, prevPost)
if err != nil {
summary := ""
for _, ru := range line {
summary += string(ru)
if len(summary) > 60 {
summary += "..."
break
}
}
return nil, fmt.Errorf("Failed to parse post: %s", summary)
}
posts = append(posts, post)
prevPost = post
prev = line
}
// No posts means there's a problem.
if len(posts) == 0 {
return nil, fmt.Errorf("Session %d (%s) has no posts (too many dates?)", i+1, dateStr)
}
// Add it.
results = append(results, ParsedLog{
Log: models.Log{Date: posts[0].Time},
Posts: posts,
})
}
return results, nil
}
-151
View File
@@ -1,151 +0,0 @@
package forumlog_test
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"git.aiterp.net/rpdata/api/internal/importers/forumlog"
"git.aiterp.net/rpdata/api/models"
)
func TestParseLogs(t *testing.T) {
results, err := forumlog.ParseLogs(testLog, time.UTC)
if err != nil {
t.Fatalf("Parse: %s", err)
}
assert.Equal(t, 2, len(results), "Amount of results.")
assert.Equal(t, testLogPosts[0], results[0].Posts, "First log's posts.")
assert.Equal(t, testLogPosts[1], results[1].Posts, "Second log's posts.")
}
func TestParseLogsErrors(t *testing.T) {
_, err1 := forumlog.ParseLogs(brokenLogNoPosts, time.UTC)
_, err2 := forumlog.ParseLogs(brokenLogBrokenPost, time.UTC)
_, err3 := forumlog.ParseLogs(brokenLogBrokenDate, time.UTC)
t.Log("Should be about no posts:", err1)
t.Log("Should be about a broken post:", err2)
t.Log("Should be about a broken date:", err3)
assert.NotEqual(t, err1, nil, "Should be no posts")
assert.NotEqual(t, err2, nil, "Should be about a bad post")
assert.NotEqual(t, err3, nil, "Should be about a broken date")
}
var testLog = `
Date:
July 25, 2013
July 26, 2013
GM:
Tyranniac
Length:
92 posts
Characters:
Fera'Sel nar Veltar - Baphomet
Renala T’Iavay - Gisle
Steven Briggs - Dante
NPCs:
Receptionist
Relevant RPs:
<-- Hub - Miner's Respite (July 16)
--> Event - Hinpinn's Salvage Mission (August 5)
[21:28] * @Tyranniac | The Hnipinn Minerals local administrative center is a rather small building located next to the refinery. The area is mostly asphalt and industrial surroundings. The gate in the fence surrounding the refinery is nearby, with a transport truck just being admitted by the guard - rather heavily armed for corporate security. Despite not being that late, it's rather dark due to the rain-bearing clouds that have just started emptying their content. The windows of the office glow with invitingly warm light. A receptionist can be seen working through the glass door.
[21:46] * Steve_Briggs approaches the building at a brisk jog, pulling up the collar of his jacket to block as much of the rain as possible. Stopping just short of the door, he holds back and awaits Renala. "Well, hopefully this is the universe's way of getting our bad luck out of the way early." he says, glancing up at the sky. "Good things to come!" he adds with a smile.
[21:46] * Steve_Briggs approaches the building at a brisk jog, pulling up the collar of his jacket to block as much of the rain as possible. Stopping just short of the door, he holds back and awaits Renala. "Well, hopefully this is the universe's way of getting our bad luck out of the way early." he says, glancing up at the sky. "Good things to come!" he adds with a smile.
--> Stuff and things
<-- Stuff and things
[21:46] * Steve_Briggs approaches the building at a brisk jog, pulling up the collar of his jacket to block as much of the rain as possible. Stopping just short of the door, he holds back and awaits Renala. "Well, hopefully this is the universe's way of getting our bad luck out of the way early." he says, glancing up at the sky. "Good things to come!" he adds with a smile.
`
var testLogPosts = [][]models.Post{
{
{
ID: "UNASSIGNED",
LogID: "UNASSIGNED",
Time: parseDate(nil, "2013-07-25 21:28:00"),
Kind: "action",
Nick: "Tyranniac",
Text: "| The Hnipinn Minerals local administrative center is a rather small building located next to the refinery. The area is mostly asphalt and industrial surroundings. The gate in the fence surrounding the refinery is nearby, with a transport truck just being admitted by the guard - rather heavily armed for corporate security. Despite not being that late, it's rather dark due to the rain-bearing clouds that have just started emptying their content. The windows of the office glow with invitingly warm light. A receptionist can be seen working through the glass door.",
Position: 1,
},
{
ID: "UNASSIGNED",
LogID: "UNASSIGNED",
Time: parseDate(nil, "2013-07-25 21:46:00"),
Kind: "action",
Nick: "Steve_Briggs",
Text: `approaches the building at a brisk jog, pulling up the collar of his jacket to block as much of the rain as possible. Stopping just short of the door, he holds back and awaits Renala. "Well, hopefully this is the universe's way of getting our bad luck out of the way early." he says, glancing up at the sky. "Good things to come!" he adds with a smile.`,
Position: 2,
},
},
{
{
ID: "UNASSIGNED",
LogID: "UNASSIGNED",
Time: parseDate(nil, "2013-07-26 21:46:00"),
Kind: "action",
Nick: "Steve_Briggs",
Text: `approaches the building at a brisk jog, pulling up the collar of his jacket to block as much of the rain as possible. Stopping just short of the door, he holds back and awaits Renala. "Well, hopefully this is the universe's way of getting our bad luck out of the way early." he says, glancing up at the sky. "Good things to come!" he adds with a smile.`,
Position: 1,
},
{
ID: "UNASSIGNED",
LogID: "UNASSIGNED",
Time: parseDate(nil, "2013-07-26 21:46:00"),
Kind: "action",
Nick: "Steve_Briggs",
Text: `approaches the building at a brisk jog, pulling up the collar of his jacket to block as much of the rain as possible. Stopping just short of the door, he holds back and awaits Renala. "Well, hopefully this is the universe's way of getting our bad luck out of the way early." he says, glancing up at the sky. "Good things to come!" he adds with a smile.`,
Position: 2,
},
},
}
var brokenLogNoPosts = `
Date:
September 27, 2014
`
var brokenLogBrokenPost = `
Date:
September 10, 2038
[12:38] * Some_Character is lasered by a reaper, and turns to ash before even knowing what happened.
[12:4 * Some_Other_Character also meets the same fate, because the Reapers are no joke.
`
var brokenLogBrokenDate = `
Date:
10 September, 2038
[12:34] * =Scene= Stuff happens.
`
func parseDate(t *testing.T, date string) time.Time {
result, err := time.Parse("2006-01-02 15:04:05", date)
if err != nil {
if t != nil {
t.Fatal("Could not parse date", date, err)
} else {
panic("Could not parse date: " + err.Error())
}
}
return result
}
-35
View File
@@ -1,35 +0,0 @@
package forumlog
import (
"bufio"
"strings"
)
// ParseMetadata parses metadata, discards the broken parts, and returns the
// parsed data as a map (`m`) and the position of the first IRC post (`n`)
func ParseMetadata(data string) map[string][]string {
result := make(map[string][]string)
key := ""
scanner := bufio.NewScanner(strings.NewReader(data))
for scanner.Scan() {
line := strings.Trim(scanner.Text(), "\r\t  ")
if strings.HasPrefix(line, "[") {
break
}
if len(line) < 1 {
key = ""
continue
}
if key == "" {
split := strings.Split(line, ":")
key = split[0]
} else {
result[key] = append(result[key], line)
}
}
return result
}
@@ -1,81 +0,0 @@
package forumlog_test
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
"git.aiterp.net/rpdata/api/internal/importers/forumlog"
)
func TestParseMetadata(t *testing.T) {
data := forumlog.ParseMetadata(testData1)
assert.Equal(t, data["Date"], []string{
"August 11, 2014",
"August 12, 2014",
"August 13, 2014",
})
assert.Equal(t, data["Location"], []string{
"Aroste, an island off the Derraian coastline.",
})
assert.Equal(t, data["GM"], []string{
"Gisle",
})
assert.Equal(t, data["Length"], []string{
"847 posts",
})
assert.Equal(t, len(data["Stuff"]), 0)
for key := range data {
if strings.HasPrefix(key, "[") {
t.Error("It should have stopped at the IRC post, yet this key exists:", key)
}
}
}
var testData1 = `
Date: August 2185
August 11, 2014
August 12, 2014
August 13, 2014
Location:
Aroste, an island off the Derraian coastline.
GM:
Gisle
Length:
847 posts
Characters:
Calyx Vadris - Osiris
Damien Monroe - Bowe
Jason Wolfe - Dante
Renala T'Iavay - Gisle
Victoria Steels - MCB280
NPCs:
Recurring:
Halisi - Tyranniac
Jattic - Dante
Jelvan Darennon - Gisle
Leah - Dante
Marissa T'Evin - Gisle
Philip Lacour - Tyranniac
One-off:
Reidas Falten (Turian - Guard meeting them on their trip to the building)
Sarian T'Dera (Asari - Asari opening fire on Calyx, Leah and Renala)
Mariam Adams (Human - The receptionist.)
Prerix Falten (Turian2 - Victoria's chair)
Jonathan Lyng (OtherHuman - The one opening fire on Leah entering the garage)
Alerena Teris (Teris - Asari met in the maintenance corridor)
<-- Redorck Agency - August 11, 2014
[13:19] * Leah looks over at Damien. "Haven't seen you around the agency." she says curiously, "New hire?"
`
-48
View File
@@ -1,48 +0,0 @@
package mirclike
import (
"errors"
"strings"
"time"
"git.aiterp.net/rpdata/api/models"
)
// ErrEmptyLog is returned by ParseLog if there are no (valid) posts in the log.
var ErrEmptyLog = errors.New("No valid posts found in log")
// ParseLog parses the log and returns the things that can be gleamed from them.
func ParseLog(data string, date time.Time, strict bool) (models.Log, []models.Post, error) {
lines := strings.Split(data, "\n")
posts := make([]models.Post, 0, len(lines))
prev := models.Post{}
for _, line := range lines {
line = strings.Trim(line, "\r\t  ")
if len(line) < 1 {
continue
}
post, err := ParsePost(line, date, prev)
if err != nil {
if strict {
return models.Log{}, nil, err
}
continue
}
posts = append(posts, post)
prev = post
}
if len(posts) == 0 {
return models.Log{}, nil, ErrEmptyLog
}
log := models.Log{
Date: posts[0].Time,
}
return log, posts, nil
}
-69
View File
@@ -1,69 +0,0 @@
package mirclike_test
import (
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
"git.aiterp.net/rpdata/api/internal/importers/mirclike"
"git.aiterp.net/rpdata/api/models"
)
var testLog = strings.Join([]string{
"[11:21] * Va`ynna_Atana returns to the apartment at the end of another long day. She hangs up her jacket and a green scarf next to the door before going straight to the bathroom; she leaves the door open, though. She walked home with Uvena this time, but the two parted ways downstairs.",
"",
"[11:21] <=Scene=> The weather outside is not pleasant in the slightest. The storm is still going on, visibility is reduced and the worst gusts of wind can be felt inside the apartment as a faint shudder. ",
"",
"[11:",
" [11:27] <Test> Stuff and things.",
}, "\r\n")
var testLogPosts = []models.Post{
{
ID: "UNASSIGNED",
LogID: "UNASSIGNED",
Time: parseDate(nil, "2018-05-11 11:21:00"),
Kind: "action",
Nick: "Va`ynna_Atana",
Position: 1,
Text: "returns to the apartment at the end of another long day. She hangs up her jacket and a green scarf next to the door before going straight to the bathroom; she leaves the door open, though. She walked home with Uvena this time, but the two parted ways downstairs.",
},
{
ID: "UNASSIGNED",
LogID: "UNASSIGNED",
Time: parseDate(nil, "2018-05-11 11:21:00"),
Kind: "scene",
Nick: "=Scene=",
Position: 2,
Text: "The weather outside is not pleasant in the slightest. The storm is still going on, visibility is reduced and the worst gusts of wind can be felt inside the apartment as a faint shudder.",
},
{
ID: "UNASSIGNED",
LogID: "UNASSIGNED",
Time: parseDate(nil, "2018-05-11 11:27:00"),
Kind: "text",
Nick: "Test",
Position: 3,
Text: "Stuff and things.",
},
}
func TestParseLog(t *testing.T) {
log, posts, err := mirclike.ParseLog(testLog, parseDate(t, "2018-05-11 00:00:00"), false)
if err != nil {
t.Fatal("ParseLog", err)
}
assert.Equal(t, testLogPosts, posts)
assert.Equal(t, posts[0].Time, log.Date, "Log's date should be the first post's.")
}
func TestParseLogErrors(t *testing.T) {
_, _, err1 := mirclike.ParseLog("\n\n\n\n\n \n\t\r\n", time.Time{}, false)
_, _, err2 := mirclike.ParseLog("\n\n\n\n\n[14:57]* Stuff \n\t\r\n", time.Time{}, true)
assert.Equal(t, mirclike.ErrEmptyLog, err1)
assert.Equal(t, mirclike.ErrNotPost, err2)
}
-98
View File
@@ -1,98 +0,0 @@
package mirclike
import (
"errors"
"strconv"
"strings"
"time"
"git.aiterp.net/rpdata/api/models"
)
// ErrNotPost is returned by parsePost if the line is empty or not a post.
var ErrNotPost = errors.New("not a post")
// ParsePost parses a post from a mirc-like line. If the previous post is included (it can be empty), it will be used
// to determine whether midnight has passed.
func ParsePost(line string, date time.Time, prev models.Post) (models.Post, error) {
// Do basic validation
line = strings.Trim(line, "  \t\n\r")
if len(line) == 0 || !strings.HasPrefix(line, "[") {
return models.Post{}, ErrNotPost
}
// Parse timestamp
tsEndIndex := strings.IndexByte(line, ']')
if tsEndIndex == -1 || len(line) < tsEndIndex+5 {
return models.Post{}, ErrNotPost
}
tsStr := line[1:tsEndIndex]
tsSplit := strings.Split(tsStr, ":")
tsUnits := make([]int, len(tsSplit))
if len(tsSplit) < 2 {
return models.Post{}, ErrNotPost
}
for i := range tsSplit {
n, err := strconv.Atoi(tsSplit[i])
if err != nil {
return models.Post{}, ErrNotPost
}
tsUnits[i] = n
}
if len(tsUnits) == 2 {
tsUnits = append(tsUnits, 0)
}
// Determine timestamp from parsed data and previous post.
ts := time.Date(date.Year(), date.Month(), date.Day(), tsUnits[0], tsUnits[1], tsUnits[2], 0, date.Location())
if !prev.Time.IsZero() && prev.Time.Sub(ts) > 30*time.Minute {
ts = time.Date(prev.Time.Year(), prev.Time.Month(), prev.Time.Day()+1, tsUnits[0], tsUnits[1], tsUnits[2], 0, date.Location())
}
if line[tsEndIndex+2] == '*' {
split := strings.SplitN(line[tsEndIndex+4:], " ", 2)
if len(split) == 1 {
return models.Post{}, ErrNotPost
}
post := models.Post{
ID: "UNASSIGNED",
LogID: "UNASSIGNED",
Time: ts,
Kind: "action",
Nick: strings.TrimLeft(split[0], "+@!~"),
Text: split[1],
Position: prev.Position + 1,
}
if post.Nick[0] == '=' {
post.Kind = "scene"
}
return post, nil
} else if line[tsEndIndex+2] == '<' {
split := strings.SplitN(line[tsEndIndex+2:], " ", 2)
if len(split) == 1 {
return models.Post{}, ErrNotPost
}
post := models.Post{
ID: "UNASSIGNED",
LogID: "UNASSIGNED",
Time: ts,
Kind: "text",
Nick: strings.TrimLeft(split[0][1:len(split[0])-1], "+@!~"),
Text: split[1],
Position: prev.Position + 1,
}
if post.Nick[0] == '=' {
post.Kind = "scene"
}
return post, nil
} else {
return models.Post{}, ErrNotPost
}
}
-130
View File
@@ -1,130 +0,0 @@
package mirclike_test
import (
"fmt"
"testing"
"time"
"github.com/stretchr/testify/assert"
"git.aiterp.net/rpdata/api/internal/importers/mirclike"
"git.aiterp.net/rpdata/api/models"
)
func TestParsePost(t *testing.T) {
table := []struct {
Input string
TS string
Kind string
Nick string
Text string
}{
{
"[12:34] * Stuff does things.",
"12:34:00", "action", "Stuff", "does things.",
},
{
"[12:34] <Stuff> Things said.",
"12:34:00", "text", "Stuff", "Things said.",
},
{
"[13:36:59] <Stuff> Things said.",
"13:36:59", "text", "Stuff", "Things said.",
},
{
"[23:59] <=Scene=> Scenery and such.",
"23:59:00", "scene", "=Scene=", "Scenery and such.",
},
{
"[01:10:11] * =Scene= Scenery and such from the forum or mIRC using my old script.",
"01:10:11", "scene", "=Scene=", "Scenery and such from the forum or mIRC using my old script.",
},
}
for i, row := range table {
t.Run(fmt.Sprintf("Row_%d", i), func(t *testing.T) {
post, err := mirclike.ParsePost(row.Input, time.Now(), models.Post{})
if err != nil {
t.Fatal("Could not parse post:", err)
}
assert.Equal(t, row.TS, post.Time.Format("15:04:05"), "Timestamps should match.")
assert.Equal(t, row.Kind, post.Kind, "Kinds should match.")
assert.Equal(t, row.Nick, post.Nick, "Kinds should match.")
assert.Equal(t, row.Text, post.Text, "Kinds should match.")
})
}
}
func TestParsePostErrors(t *testing.T) {
table := []struct {
Input string
Err error
}{
{"[12:34] <Stuff> Things said.", nil},
{"[12:34] >Stuff> Things said.", mirclike.ErrNotPost},
{"12:34] <Stuff> Things said.", mirclike.ErrNotPost},
{"* Stuff Things said.", mirclike.ErrNotPost},
{"", mirclike.ErrNotPost},
{"[12:34 <Stuff> Things said.", mirclike.ErrNotPost},
{"[TE:XT] <Stuff> Things said.", mirclike.ErrNotPost},
{"[10] <Stuff> Things said.", mirclike.ErrNotPost},
{"[12:34:56:789] <Stuff> Things said.", nil},
{"[12:34:56.789] <Stuff> Things said.", mirclike.ErrNotPost},
{"[12:34] <Stuff>", mirclike.ErrNotPost},
{"[12:34] * Stuff", mirclike.ErrNotPost},
{"[12:34] =Scene=", mirclike.ErrNotPost},
{"[12:34] <=Scene=>", mirclike.ErrNotPost},
}
for i, row := range table {
t.Run(fmt.Sprintf("Row_%d", i), func(t *testing.T) {
_, err := mirclike.ParsePost(row.Input, time.Now(), models.Post{})
assert.Equal(t, row.Err, err, "Error should match")
})
}
}
func TestParseNextDay(t *testing.T) {
table := []struct {
Prev time.Time
TS string
Time time.Time
}{
{Prev: parseDate(t, "2019-01-12 12:00:00"), TS: "12:01", Time: parseDate(t, "2019-01-12 12:01:00")},
{Prev: parseDate(t, "2019-01-12 12:00:00"), TS: "11:53:13", Time: parseDate(t, "2019-01-12 11:53:13")},
{Prev: parseDate(t, "2019-04-08 23:51:59"), TS: "00:09", Time: parseDate(t, "2019-04-09 00:09:00")},
{Prev: parseDate(t, "2019-01-12 12:00:00"), TS: "11:29:59", Time: parseDate(t, "2019-01-13 11:29:59")},
}
for i, row := range table {
t.Run(fmt.Sprintf("Row_%d", i), func(t *testing.T) {
input := fmt.Sprintf("[%s] * Stuff does things.", row.TS)
post, err := mirclike.ParsePost(input, row.Prev, models.Post{Time: row.Prev})
if err != nil {
t.Fatal("Could not parse post:", err)
}
assert.Equal(t, row.Time, post.Time)
})
}
}
func parseDate(t *testing.T, date string) time.Time {
result, err := time.Parse("2006-01-02 15:04:05", date)
if err != nil {
if t != nil {
t.Fatal("Could not parse date", date, err)
} else {
panic("Could not parse date: " + err.Error())
}
}
return result
}
func formatDate(date time.Time) string {
return date.UTC().Format("2006-01-02 15:04:05")
}