logs: Added forumlog parsers.

internal/importers/forumlog/logs.go

@@ -0,0 +1,101 @@
+package forumlog
+
+import (
+	"bufio"
+	"fmt"
+	"strings"
+	"time"
+
+	"git.aiterp.net/rpdata/api/internal/importers/mirclike"
+
+	"git.aiterp.net/rpdata/api/models"
+)
+
+// A ParsedLog contains the parsed log header and its posts.
+type ParsedLog struct {
+	Log   models.Log
+	Posts []models.Post
+}
+
+// ParseLogs parses the logs from the data.
+func ParseLogs(data string) ([]ParsedLog, error) {
+	metadata := ParseMetadata(data)
+	results := make([]ParsedLog, 0, len(metadata["Date"]))
+	scanner := bufio.NewScanner(strings.NewReader(data))
+
+	for i, dateStr := range metadata["Date"] {
+		// Parse date
+		date, err := time.Parse("January 2, 2006", dateStr)
+		if err != nil {
+			return nil, fmt.Errorf("Failed to parse date #%d: %#+v is not the in the correct format of \"January 2, 2006\"", i+1, dateStr)
+		}
+
+		// Parse posts
+		posts := make([]models.Post, 0, 128)
+		parsing := false
+		prev := ""
+		prevPost := models.Post{}
+		for scanner.Scan() {
+			line := strings.Trim(scanner.Text(), "\r\t  ")
+
+			// Skip lines with links to other logs using the --> and <-- notation.
+			if strings.HasPrefix(line, "-") || strings.HasPrefix(line, "<") {
+				prev = line
+				continue
+			}
+
+			// If parsing and reaching a double empty-line, the session is done.
+			if parsing && len(line) < 2 && len(prev) < 2 {
+				break
+			}
+
+			// Skip empty lines, but record them as thep revious for the above check.
+			if len(line) < 2 {
+				prev = line
+				continue
+			}
+
+			// If not parsing, skip until the first mirclike post.
+			if !parsing {
+				if strings.HasPrefix(line, "[") {
+					parsing = true
+				} else {
+					prev = line
+					continue
+				}
+			}
+
+			// Parse the post.
+			post, err := mirclike.ParsePost(line, date, prevPost)
+			if err != nil {
+				summary := ""
+				for _, ru := range line {
+					summary += string(ru)
+					if len(summary) > 60 {
+						summary += "..."
+						break
+					}
+				}
+
+				return nil, fmt.Errorf("Failed to parse post: %s", summary)
+			}
+
+			posts = append(posts, post)
+			prevPost = post
+			prev = line
+		}
+
+		// No posts means there's a problem.
+		if len(posts) == 0 {
+			return nil, fmt.Errorf("Session %d (%s) has no posts (too many dates?)", i+1, dateStr)
+		}
+
+		// Add it.
+		results = append(results, ParsedLog{
+			Log:   models.Log{Date: posts[0].Time},
+			Posts: posts,
+		})
+	}
+
+	return results, nil
+}

internal/importers/forumlog/logs_test.go

@@ -0,0 +1,151 @@
+package forumlog_test
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	"git.aiterp.net/rpdata/api/internal/importers/forumlog"
+
+	"git.aiterp.net/rpdata/api/models"
+)
+
+func TestParseLogs(t *testing.T) {
+	results, err := forumlog.ParseLogs(testLog)
+	if err != nil {
+		t.Fatalf("Parse: %s", err)
+	}
+
+	assert.Equal(t, 2, len(results), "Amount of results.")
+	assert.Equal(t, testLogPosts[0], results[0].Posts, "First log's posts.")
+	assert.Equal(t, testLogPosts[1], results[1].Posts, "Second log's posts.")
+}
+
+func TestParseLogsErrors(t *testing.T) {
+	_, err1 := forumlog.ParseLogs(brokenLogNoPosts)
+	_, err2 := forumlog.ParseLogs(brokenLogBrokenPost)
+	_, err3 := forumlog.ParseLogs(brokenLogBrokenDate)
+
+	t.Log("Should be about no posts:", err1)
+	t.Log("Should be about a broken post:", err2)
+	t.Log("Should be about a broken date:", err3)
+
+	assert.NotEqual(t, err1, nil, "Should be no posts")
+	assert.NotEqual(t, err2, nil, "Should be about a bad post")
+	assert.NotEqual(t, err3, nil, "Should be about a broken date")
+}
+
+var testLog = `
+Date:
+July 25, 2013
+July 26, 2013
+
+GM:
+Tyranniac
+
+Length:
+92 posts
+
+Characters:
+Fera'Sel nar Veltar - Baphomet
+Renala T’Iavay - Gisle
+Steven Briggs - Dante
+
+NPCs:
+Receptionist
+
+Relevant RPs:
+<-- Hub - Miner's Respite (July 16)
+--> Event - Hinpinn's Salvage Mission (August 5)
+
+ [21:28] * @Tyranniac | The Hnipinn Minerals local administrative center is a rather small building located next to the refinery. The area is mostly asphalt and industrial surroundings. The gate in the fence surrounding the refinery is nearby, with a transport truck just being admitted by the guard - rather heavily armed for corporate security. Despite not being that late, it's rather dark due to the rain-bearing clouds that have just started emptying their content. The windows of the office glow with invitingly warm light. A receptionist can be seen working through the glass door.
+
+ [21:46] * Steve_Briggs approaches the building at a brisk jog, pulling up the collar of his jacket to block as much of the rain as possible. Stopping just short of the door, he holds back and awaits Renala. "Well, hopefully this is the universe's way of getting our bad luck out of the way early." he says, glancing up at the sky. "Good things to come!" he adds with a smile.
+
+
+ [21:46] * Steve_Briggs approaches the building at a brisk jog, pulling up the collar of his jacket to block as much of the rain as possible. Stopping just short of the door, he holds back and awaits Renala. "Well, hopefully this is the universe's way of getting our bad luck out of the way early." he says, glancing up at the sky. "Good things to come!" he adds with a smile.
+
+ --> Stuff and things
+
+ <-- Stuff and things
+
+ [21:46] * Steve_Briggs approaches the building at a brisk jog, pulling up the collar of his jacket to block as much of the rain as possible. Stopping just short of the door, he holds back and awaits Renala. "Well, hopefully this is the universe's way of getting our bad luck out of the way early." he says, glancing up at the sky. "Good things to come!" he adds with a smile.
+ `
+
+var testLogPosts = [][]models.Post{
+	{
+		{
+			ID:       "UNASSIGNED",
+			LogID:    "UNASSIGNED",
+			Time:     parseDate(nil, "2013-07-25 21:28:00"),
+			Kind:     "action",
+			Nick:     "Tyranniac",
+			Text:     "| The Hnipinn Minerals local administrative center is a rather small building located next to the refinery. The area is mostly asphalt and industrial surroundings. The gate in the fence surrounding the refinery is nearby, with a transport truck just being admitted by the guard - rather heavily armed for corporate security. Despite not being that late, it's rather dark due to the rain-bearing clouds that have just started emptying their content. The windows of the office glow with invitingly warm light. A receptionist can be seen working through the glass door.",
+			Position: 1,
+		},
+		{
+			ID:       "UNASSIGNED",
+			LogID:    "UNASSIGNED",
+			Time:     parseDate(nil, "2013-07-25 21:46:00"),
+			Kind:     "action",
+			Nick:     "Steve_Briggs",
+			Text:     `approaches the building at a brisk jog, pulling up the collar of his jacket to block as much of the rain as possible. Stopping just short of the door, he holds back and awaits Renala. "Well, hopefully this is the universe's way of getting our bad luck out of the way early." he says, glancing up at the sky. "Good things to come!" he adds with a smile.`,
+			Position: 2,
+		},
+	},
+	{
+		{
+			ID:       "UNASSIGNED",
+			LogID:    "UNASSIGNED",
+			Time:     parseDate(nil, "2013-07-26 21:46:00"),
+			Kind:     "action",
+			Nick:     "Steve_Briggs",
+			Text:     `approaches the building at a brisk jog, pulling up the collar of his jacket to block as much of the rain as possible. Stopping just short of the door, he holds back and awaits Renala. "Well, hopefully this is the universe's way of getting our bad luck out of the way early." he says, glancing up at the sky. "Good things to come!" he adds with a smile.`,
+			Position: 1,
+		},
+		{
+			ID:       "UNASSIGNED",
+			LogID:    "UNASSIGNED",
+			Time:     parseDate(nil, "2013-07-26 21:46:00"),
+			Kind:     "action",
+			Nick:     "Steve_Briggs",
+			Text:     `approaches the building at a brisk jog, pulling up the collar of his jacket to block as much of the rain as possible. Stopping just short of the door, he holds back and awaits Renala. "Well, hopefully this is the universe's way of getting our bad luck out of the way early." he says, glancing up at the sky. "Good things to come!" he adds with a smile.`,
+			Position: 2,
+		},
+	},
+}
+
+var brokenLogNoPosts = `
+Date:
+September 27, 2014
+`
+
+var brokenLogBrokenPost = `
+Date:
+September 10, 2038
+
+[12:38] * Some_Character is lasered by a reaper, and turns to ash before even knowing what happened.
+
+[12:4 * Some_Other_Character also meets the same fate, because the Reapers are no joke.
+`
+
+var brokenLogBrokenDate = `
+Date:
+10 September, 2038
+
+[12:34] * =Scene= Stuff happens.
+`
+
+func parseDate(t *testing.T, date string) time.Time {
+	result, err := time.Parse("2006-01-02 15:04:05", date)
+	if err != nil {
+		if t != nil {
+			t.Fatal("Could not parse date", date, err)
+		} else {
+			panic("Could not parse date: " + err.Error())
+		}
+	}
+
+	return result
+}

internal/importers/forumlog/metadata.go

@@ -0,0 +1,35 @@
+package forumlog
+
+import (
+	"bufio"
+	"strings"
+)
+
+// ParseMetadata parses metadata, discards the broken parts, and returns the
+// parsed data as a map (`m`) and the position of the first IRC post (`n`)
+func ParseMetadata(data string) map[string][]string {
+	result := make(map[string][]string)
+	key := ""
+
+	scanner := bufio.NewScanner(strings.NewReader(data))
+	for scanner.Scan() {
+		line := strings.Trim(scanner.Text(), "\r\t  ")
+
+		if strings.HasPrefix(line, "[") {
+			break
+		}
+		if len(line) < 1 {
+			key = ""
+			continue
+		}
+
+		if key == "" {
+			split := strings.Split(line, ":")
+			key = split[0]
+		} else {
+			result[key] = append(result[key], line)
+		}
+	}
+
+	return result
+}

internal/importers/forumlog/metadata_test.go

@@ -0,0 +1,81 @@
+package forumlog_test
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"git.aiterp.net/rpdata/api/internal/importers/forumlog"
+)
+
+func TestParseMetadata(t *testing.T) {
+	data := forumlog.ParseMetadata(testData1)
+
+	assert.Equal(t, data["Date"], []string{
+		"August 11, 2014",
+		"August 12, 2014",
+		"August 13, 2014",
+	})
+	assert.Equal(t, data["Location"], []string{
+		"Aroste, an island off the Derraian coastline.",
+	})
+	assert.Equal(t, data["GM"], []string{
+		"Gisle",
+	})
+	assert.Equal(t, data["Length"], []string{
+		"847 posts",
+	})
+	assert.Equal(t, len(data["Stuff"]), 0)
+
+	for key := range data {
+		if strings.HasPrefix(key, "[") {
+			t.Error("It should have stopped at the IRC post, yet this key exists:", key)
+		}
+	}
+}
+
+var testData1 = `
+Date: August 2185
+August 11, 2014
+August 12, 2014
+August 13, 2014
+
+Location:
+Aroste, an island off the Derraian coastline.
+
+GM:
+Gisle
+
+Length:
+847 posts
+
+Characters:
+Calyx Vadris - Osiris
+Damien Monroe - Bowe
+Jason Wolfe - Dante
+Renala T'Iavay - Gisle
+Victoria Steels - MCB280
+
+NPCs:
+Recurring:
+Halisi - Tyranniac
+Jattic - Dante
+Jelvan Darennon - Gisle
+Leah - Dante
+Marissa T'Evin - Gisle
+Philip Lacour - Tyranniac
+
+One-off:
+Reidas Falten (Turian - Guard meeting them on their trip to the building)
+Sarian T'Dera (Asari - Asari opening fire on Calyx, Leah and Renala)
+Mariam Adams (Human - The receptionist.)
+Prerix Falten (Turian2 - Victoria's chair)
+Jonathan Lyng (OtherHuman - The one opening fire on Leah entering the garage)
+Alerena Teris (Teris - Asari met in the maintenance corridor)
+
+<-- Redorck Agency - August 11, 2014
+
+[13:19] * Leah looks over at Damien. "Haven't seen you around the agency." she says curiously, "New hire?"
+
+`