rpdata-lb2logimport: Updated it to fix build errors related to change in logs.UpdateChracters.

Dockerfile

@@ -7,28 +7,27 @@ WORKDIR /go/src/git.aiterp.net/rpdata/api/
 COPY . .
 RUN rm -rf ./vendor
 
-# Load build tools and dependencies
+# Install build tools and dependencies
 RUN go get -u github.com/golang/dep/cmd/dep
 RUN go get -u github.com/vektah/gorunpkg
-RUN go get -u github.com/jteeuwen/go-bindata/...
 RUN dep ensure
 RUN go generate ./...
 
 # Build server and tools (CGO disabled to make them 100% static)
 ENV CGO_ENABLED 0
-RUN go build -a -installsuffix cgo -ldflags="-s -w" -o /binaries/rpdata-server ./cmd/rpdata-server
-RUN go build -a -installsuffix cgo -ldflags="-s -w" -o /binaries/rpdata-lb2charimport ./cmd/rpdata-lb2charimport
-RUN go build -a -installsuffix cgo -ldflags="-s -w" -o /binaries/rpdata-lb2logimport ./cmd/rpdata-lb2logimport
-# TODO: RUN go build -a -installsuffix cgo -ldflags="-s -w" -o /binaries/rpdata-wikifileimport ./cmd/rpdata-wikifileimport
-RUN go build -a -installsuffix cgo -ldflags="-s -w" -o /binaries/rpdata-ensurechannels ./cmd/rpdata-ensurechannels
-RUN go build -a -installsuffix cgo -ldflags="-s -w" -o /binaries/rpdata-as2storyimport ./cmd/rpdata-as2storyimport
+RUN go build -installsuffix cgo -ldflags="-s -w" -o /binaries/rpdata-server ./cmd/rpdata-server
+RUN go build -installsuffix cgo -ldflags="-s -w" -o /binaries/rpdata-lb2charimport ./cmd/rpdata-lb2charimport
+RUN go build -installsuffix cgo -ldflags="-s -w" -o /binaries/rpdata-lb2logimport ./cmd/rpdata-lb2logimport
+# TODO: RUN go build -installsuffix cgo -ldflags="-s -w" -o /binaries/rpdata-wikifileimport ./cmd/rpdata-wikifileimport
+RUN go build -installsuffix cgo -ldflags="-s -w" -o /binaries/rpdata-ensurechannels ./cmd/rpdata-ensurechannels
+RUN go build -installsuffix cgo -ldflags="-s -w" -o /binaries/rpdata-as2storyimport ./cmd/rpdata-as2storyimport
 
 ## 2. Distribute
 # Use alpine linux
 FROM alpine:3.7
 
-# Install certificates to enable HTTPS GET requests
-RUN apk add --no-cache ca-certificates
+# Install certificates to enable HTTPS GET requests, and tzdata for log imports.
+RUN apk add --no-cache ca-certificates tzdata
 
 # Copy build files into final container
 COPY --from=build-server /binaries/* /usr/local/bin/

Gopkg.lock

@@ -23,6 +23,12 @@
   revision = "3d21ba515fe27b856f230847e856431ae1724adc"
   version = "v1.0.0"
 
+[[projects]]
+  name = "github.com/davecgh/go-spew"
+  packages = ["spew"]
+  revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73"
+  version = "v1.1.1"
+
 [[projects]]
   name = "github.com/dgrijalva/jwt-go"
   packages = ["."]
@@ -71,27 +77,6 @@
   revision = "78139374585c29dcb97b8f33089ed11959e4be59"
   version = "v5"
 
-[[projects]]
-  branch = "master"
-  name = "github.com/graph-gophers/graphql-go"
-  packages = [
-    ".",
-    "errors",
-    "internal/common",
-    "internal/exec",
-    "internal/exec/packer",
-    "internal/exec/resolvable",
-    "internal/exec/selected",
-    "internal/query",
-    "internal/schema",
-    "internal/validation",
-    "introspection",
-    "log",
-    "relay",
-    "trace"
-  ]
-  revision = "9ebf33af539ab8cb832c7107bc0a978ca8dbc0de"
-
 [[projects]]
   name = "github.com/hashicorp/golang-lru"
   packages = [
@@ -133,7 +118,6 @@
   name = "github.com/opentracing/opentracing-go"
   packages = [
     ".",
-    "ext",
     "log"
   ]
   revision = "1949ddbfd147afd4d964a9f00b24eb291e0e7c38"
@@ -145,12 +129,24 @@
   revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
   version = "v0.8.0"
 
+[[projects]]
+  name = "github.com/pmezard/go-difflib"
+  packages = ["difflib"]
+  revision = "792786c7400a136282c1664665ae0a8db921c6c2"
+  version = "v1.0.0"
+
 [[projects]]
   name = "github.com/sirupsen/logrus"
   packages = ["."]
   revision = "c155da19408a8799da419ed3eeb0cb5db0ad5dbc"
   version = "v1.0.5"
 
+[[projects]]
+  name = "github.com/stretchr/testify"
+  packages = ["assert"]
+  revision = "f35b8ab0b5a2cef36673838d662e249dd9c94686"
+  version = "v1.2.2"
+
 [[projects]]
   name = "github.com/urfave/cli"
   packages = ["."]
@@ -249,6 +245,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "3c9430afa8b9260026925c8f496af934e2e573a2bd4587bb634fd12a20ce6c2c"
+  inputs-digest = "eb8fc099a909f5d8f756b24fbcb6cc2856e132bd3ed16b8fe512f22fabb99596"
   solver-name = "gps-cdcl"
   solver-version = 1

Gopkg.toml

@@ -39,5 +39,5 @@ required = [ "github.com/99designs/gqlgen" ]
   unused-packages = true
 
 [[constraint]]
-  branch = "master"
-  name = "github.com/graph-gophers/graphql-go"
+  name = "github.com/stretchr/testify"
+  version = "1.2.2"

cmd/rpdata-lb2logimport/main.go

@@ -100,7 +100,7 @@ func main() {
 			}
 		}
 
-		l, err = logs.UpdateCharacters(l)
+		l, err = logs.UpdateCharacters(l, nil)
 		if err != nil {
 			log.Println(err)
 		}

cmd/rpdata-server/main.go

@@ -8,6 +8,9 @@ import (
 	"runtime/debug"
 	"strings"
 
+	"git.aiterp.net/rpdata/api/models"
+	"git.aiterp.net/rpdata/api/models/changes"
+
 	"git.aiterp.net/rpdata/api/models/logs"
 
 	"git.aiterp.net/rpdata/api/graph2"
@@ -35,9 +38,39 @@ func main() {
 		log.Println("Characters updated")
 	}()
 
+	go logListedChanges()
+
 	log.Fatal(http.ListenAndServe(":8081", nil))
 }
 
+func logListedChanges() {
+	sub := changes.Subscribe(context.Background(), nil, true)
+
+	for change := range sub {
+		log.Printf("Change: Author=%#+v Model=%#+v Op=%#+v", change.Author, change.Model, change.Op)
+		for _, object := range change.Objects() {
+			switch object := object.(type) {
+			case models.Log:
+				log.Printf("\tLog: %s (%s)", object.ID, object.ShortID)
+			case models.Character:
+				log.Printf("\tCharacter: %s", object.ID)
+			case models.Channel:
+				log.Printf("\tChannel: %s", object.Name)
+			case models.Post:
+				log.Printf("\tPost: %s", object.ID)
+			case models.Story:
+				log.Printf("\tStory: %s", object.ID)
+			case models.Tag:
+				log.Printf("\tTag: (%s,%s)", object.Kind, object.Name)
+			case models.Chapter:
+				log.Printf("\tChapter: %s", object.ID)
+			}
+		}
+	}
+
+	log.Println("Change subscription closed.")
+}
+
 func queryHandler() http.HandlerFunc {
 	handler := handler.GraphQL(
 		graph2.New(),

graph2/gqlgen.yml

@@ -23,6 +23,8 @@ models:
     model: git.aiterp.net/rpdata/api/models.Channel
   ChannelsFilter:
     model: git.aiterp.net/rpdata/api/models/channels.Filter
+  UnknownNick:
+    model: git.aiterp.net/rpdata/api/models.UnknownNick
   Post:
     model: git.aiterp.net/rpdata/api/models.Post
   PostsFilter:
@@ -31,6 +33,8 @@ models:
     model: git.aiterp.net/rpdata/api/models.Log
   LogsFilter:
     model: git.aiterp.net/rpdata/api/models/logs.Filter
+  LogImporter:
+    model: git.aiterp.net/rpdata/api/models.LogImporter
   Chapter:
     model: git.aiterp.net/rpdata/api/models.Chapter
     fields:

graph2/graph.go

@@ -26,6 +26,10 @@ func (r *rootResolver) Mutation() MutationResolver {
 	return queries.MutationResolver
 }
 
+func (r *rootResolver) Subscription() SubscriptionResolver {
+	return queries.SubscriptionResolver
+}
+
 func (r *rootResolver) Log() LogResolver {
 	return &types.LogResolver
 }

graph2/queries/changes.go

@@ -2,6 +2,7 @@ package queries
 
 import (
 	"context"
+	"errors"
 
 	"git.aiterp.net/rpdata/api/models"
 	"git.aiterp.net/rpdata/api/models/changes"
@@ -12,3 +13,13 @@ import (
 func (r *resolver) Changes(ctx context.Context, filter *changes.Filter) ([]models.Change, error) {
 	return changes.List(filter)
 }
+
+/// Subscriptions
+
+func (r *subscriptionResolver) Changes(ctx context.Context, keys []models.ChangeKey) (<-chan models.Change, error) {
+	if len(keys) == 0 {
+		return nil, errors.New("At least one key is required for a subscription")
+	}
+
+	return changes.Subscribe(ctx, keys, false), nil
+}

graph2/queries/log.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"strings"
+	"time"
 
 	"git.aiterp.net/rpdata/api/models/channels"
 
@@ -96,6 +97,50 @@ func (r *mutationResolver) AddLog(ctx context.Context, input input.LogAddInput)
 	return log, nil
 }
 
+func (r *mutationResolver) ImportLog(ctx context.Context, input input.LogImportInput) ([]models.Log, error) {
+	token := auth.TokenFromContext(ctx)
+	if !token.Authenticated() || !token.Permitted("log.add") {
+		return nil, errors.New("You are not permitted to add logs")
+	}
+
+	date := time.Time{}
+	if input.Date != nil {
+		date = *input.Date
+	}
+
+	tz := time.UTC
+	if input.Timezone != nil {
+		parsedTZ, err := time.LoadLocation(*input.Timezone)
+		if err != nil {
+			return nil, errors.New("Unknown timezone: " + *input.Timezone)
+		}
+
+		tz = parsedTZ
+	}
+
+	results, err := logs.Import(input.Importer, date, tz, input.ChannelName, input.Data)
+	if err != nil {
+		return nil, err
+	}
+
+	newLogs := make([]models.Log, 0, len(results))
+	for _, result := range results {
+		go func() {
+			changes.Submit("Log", "add", token.UserID, true, changekeys.Many(result.Log), result.Log)
+			changes.Submit("Post", "add", token.UserID, true, changekeys.Many(result.Log, result.Posts), result.Posts)
+		}()
+
+		log, err := logs.UpdateCharacters(result.Log, nil)
+		if err != nil {
+			log = result.Log
+		}
+
+		newLogs = append(newLogs, log)
+	}
+
+	return newLogs, nil
+}
+
 func (r *mutationResolver) EditLog(ctx context.Context, input input.LogEditInput) (models.Log, error) {
 	token := auth.TokenFromContext(ctx)
 	if !token.Authenticated() || !token.Permitted("log.edit") {

graph2/queries/post.go

@@ -59,7 +59,7 @@ func (r *mutationResolver) AddPost(ctx context.Context, input input.PostAddInput
 		return models.Post{}, err
 	}
 
-	go logs.UpdateCharacters(log)
+	go logs.UpdateCharacters(log, nil)
 	go changes.Submit("Post", "add", token.UserID, true, changekeys.Many(log, post), post)
 
 	return post, nil
@@ -83,7 +83,7 @@ func (r *mutationResolver) EditPost(ctx context.Context, input input.PostEditInp
 				return
 			}
 
-			logs.UpdateCharacters(log)
+			logs.UpdateCharacters(log, nil)
 		}()
 	}
 
@@ -154,7 +154,7 @@ func (r *mutationResolver) RemovePost(ctx context.Context, input input.PostRemov
 			return
 		}
 
-		logs.UpdateCharacters(log)
+		logs.UpdateCharacters(log, nil)
 		changes.Submit("Post", "remove", token.UserID, true, changekeys.Many(log, post), post)
 	}()
 

graph2/queries/resolver.go

@@ -2,9 +2,13 @@ package queries
 
 type resolver struct{}
 type mutationResolver struct{}
+type subscriptionResolver struct{}
 
 // Resolver has all the queries
 var Resolver resolver
 
 // MutationResolver brings the mutagens.
 var MutationResolver *mutationResolver
+
+// SubscriptionResolver has the real-time magic.
+var SubscriptionResolver *subscriptionResolver

graph2/queries/unknownnicks.go

@@ -0,0 +1,23 @@
+package queries
+
+import (
+	"context"
+
+	"git.aiterp.net/rpdata/api/models"
+	"git.aiterp.net/rpdata/api/models/unknownnicks"
+
+	"git.aiterp.net/rpdata/api/graph2/input"
+)
+
+/// Queries
+
+func (r *resolver) UnknownNicks(ctx context.Context, filter *input.UnknownNicksFilter) ([]models.UnknownNick, error) {
+	limit := 100
+	if filter != nil {
+		if filter.Limit > 0 {
+			limit = filter.Limit
+		}
+	}
+
+	return unknownnicks.List(limit)
+}

graph2/schema/root.gql

@@ -1,6 +1,7 @@
 schema {
   query: Query
   mutation: Mutation
+  subscription: Subscription
 }
 
 type Query {
@@ -25,6 +26,10 @@ type Query {
   posts(filter: PostsFilter): [Post!]!
 
 
+  "List unknown nicks"
+  unknownNicks(filter: UnknownNicksFilter): [UnknownNick!]!
+
+
   # Find log by ID
   log(id: String!): Log!
 
@@ -95,6 +100,9 @@ type Mutation {
   # Add a new log
   addLog(input: LogAddInput!): Log!
 
+  # Import a log
+  importLog(input: LogImportInput!): [Log!]!
+
   # Edit a log
   editLog(input: LogEditInput!): Log!
 
@@ -138,6 +146,13 @@ type Mutation {
   editChannel(input: ChannelEditInput!): Channel!
 }
 
+type Subscription {
+  """
+  Changes subscribes to the changes matching the following keys.
+  """
+  changes(keys: [ChangeKeyInput!]!): Change!
+}
+
 # A Date represents a RFC3339 encoded date with up to millisecond precision.
 scalar Date
 

graph2/schema/types/Log.gql

@@ -100,4 +100,36 @@ input LogEditInput {
 input LogRemoveInput {
   # The id of the log to remove
   id: String!
+}
+
+"""
+Input for importLog mutation.
+"""
+input LogImportInput {
+  "The channel of the log, alwyas required."
+  channelName: String!
+
+  "Which importer to use."
+  importer: LogImporter!
+
+  "The timezone of the import, all log and post dates will be treated as it. It will be UTC if none is specified."
+  timezone: String
+
+  "The date of the log, if not provided in the log body."
+  date: Date
+
+  "The log body itself."
+  data: String!
+}
+
+enum LogImporter {
+  """
+  mIRC-like: This importer parses logs that copied from mIRC posts without spaces.
+  """
+  MircLike
+
+  """
+  Forum log: This importer parses the format on the forum. The displayed log, not the post source.
+  """
+  ForumLog
 }

graph2/schema/types/UnknownNick.gql

@@ -0,0 +1,20 @@
+"""
+An unknown nick is a nick found by the logbot that's not associated with
+any characters. Many are going to be false positives, but most should be
+useful in pointing out which characters are missing.
+"""
+type UnknownNick {
+  "The IRC nick that's unknown."
+  nick: String!
+
+  "The amount of logs that contains the unknown nick."
+  score: Int!
+}
+
+"""
+Input for the unknownNicks query.
+"""
+input UnknownNicksFilter {
+  "The highest amount of unknown nicks to return."
+  limit: Int!
+}

internal/counter/counter.go

@@ -30,6 +30,24 @@ func Next(category, name string) (int, error) {
 	return doc.Value, nil
 }
 
+// NextMany gets the next value of a counter, or an error if it hasn't, and increments by a specified value.
+// Any value `returned` to `returned+(increment-1)` should then be safe to use.
+func NextMany(category, name string, increment int) (int, error) {
+	id := category + "." + name
+	doc := counter{}
+
+	_, err := collection.Find(bson.M{"_id": id}).Apply(mgo.Change{
+		Update:    bson.M{"$inc": bson.M{"value": increment}},
+		Upsert:    true,
+		ReturnNew: true,
+	}, &doc)
+	if err != nil {
+		return -1, err
+	}
+
+	return doc.Value, nil
+}
+
 func init() {
 	store.HandleInit(func(db *mgo.Database) {
 		collection = db.C("core.counters")

internal/importers/forumlog/logs.go

@@ -0,0 +1,101 @@
+package forumlog
+
+import (
+	"bufio"
+	"fmt"
+	"strings"
+	"time"
+
+	"git.aiterp.net/rpdata/api/internal/importers/mirclike"
+
+	"git.aiterp.net/rpdata/api/models"
+)
+
+// A ParsedLog contains the parsed log header and its posts.
+type ParsedLog struct {
+	Log   models.Log
+	Posts []models.Post
+}
+
+// ParseLogs parses the logs from the data.
+func ParseLogs(data string, tz *time.Location) ([]ParsedLog, error) {
+	metadata := ParseMetadata(data)
+	results := make([]ParsedLog, 0, len(metadata["Date"]))
+	scanner := bufio.NewScanner(strings.NewReader(data))
+
+	for i, dateStr := range metadata["Date"] {
+		// Parse date
+		date, err := time.ParseInLocation("January 2, 2006", dateStr, tz)
+		if err != nil {
+			return nil, fmt.Errorf("Failed to parse date #%d: %#+v is not the in the correct format of \"January 2, 2006\"", i+1, dateStr)
+		}
+
+		// Parse posts
+		posts := make([]models.Post, 0, 128)
+		parsing := false
+		prev := ""
+		prevPost := models.Post{}
+		for scanner.Scan() {
+			line := strings.Trim(scanner.Text(), "\r\t  ")
+
+			// Skip lines with links to other logs using the --> and <-- notation.
+			if strings.HasPrefix(line, "-") || strings.HasPrefix(line, "<") {
+				prev = line
+				continue
+			}
+
+			// If parsing and reaching a double empty-line, the session is done.
+			if parsing && len(line) < 2 && len(prev) < 2 {
+				break
+			}
+
+			// Skip empty lines, but record them as thep revious for the above check.
+			if len(line) < 2 {
+				prev = line
+				continue
+			}
+
+			// If not parsing, skip until the first mirclike post.
+			if !parsing {
+				if strings.HasPrefix(line, "[") {
+					parsing = true
+				} else {
+					prev = line
+					continue
+				}
+			}
+
+			// Parse the post.
+			post, err := mirclike.ParsePost(line, date, prevPost)
+			if err != nil {
+				summary := ""
+				for _, ru := range line {
+					summary += string(ru)
+					if len(summary) > 60 {
+						summary += "..."
+						break
+					}
+				}
+
+				return nil, fmt.Errorf("Failed to parse post: %s", summary)
+			}
+
+			posts = append(posts, post)
+			prevPost = post
+			prev = line
+		}
+
+		// No posts means there's a problem.
+		if len(posts) == 0 {
+			return nil, fmt.Errorf("Session %d (%s) has no posts (too many dates?)", i+1, dateStr)
+		}
+
+		// Add it.
+		results = append(results, ParsedLog{
+			Log:   models.Log{Date: posts[0].Time},
+			Posts: posts,
+		})
+	}
+
+	return results, nil
+}

internal/importers/forumlog/logs_test.go

@@ -0,0 +1,151 @@
+package forumlog_test
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	"git.aiterp.net/rpdata/api/internal/importers/forumlog"
+
+	"git.aiterp.net/rpdata/api/models"
+)
+
+func TestParseLogs(t *testing.T) {
+	results, err := forumlog.ParseLogs(testLog, time.UTC)
+	if err != nil {
+		t.Fatalf("Parse: %s", err)
+	}
+
+	assert.Equal(t, 2, len(results), "Amount of results.")
+	assert.Equal(t, testLogPosts[0], results[0].Posts, "First log's posts.")
+	assert.Equal(t, testLogPosts[1], results[1].Posts, "Second log's posts.")
+}
+
+func TestParseLogsErrors(t *testing.T) {
+	_, err1 := forumlog.ParseLogs(brokenLogNoPosts, time.UTC)
+	_, err2 := forumlog.ParseLogs(brokenLogBrokenPost, time.UTC)
+	_, err3 := forumlog.ParseLogs(brokenLogBrokenDate, time.UTC)
+
+	t.Log("Should be about no posts:", err1)
+	t.Log("Should be about a broken post:", err2)
+	t.Log("Should be about a broken date:", err3)
+
+	assert.NotEqual(t, err1, nil, "Should be no posts")
+	assert.NotEqual(t, err2, nil, "Should be about a bad post")
+	assert.NotEqual(t, err3, nil, "Should be about a broken date")
+}
+
+var testLog = `
+Date:
+July 25, 2013
+July 26, 2013
+
+GM:
+Tyranniac
+
+Length:
+92 posts
+
+Characters:
+Fera'Sel nar Veltar - Baphomet
+Renala T’Iavay - Gisle
+Steven Briggs - Dante
+
+NPCs:
+Receptionist
+
+Relevant RPs:
+<-- Hub - Miner's Respite (July 16)
+--> Event - Hinpinn's Salvage Mission (August 5)
+
+ [21:28] * @Tyranniac | The Hnipinn Minerals local administrative center is a rather small building located next to the refinery. The area is mostly asphalt and industrial surroundings. The gate in the fence surrounding the refinery is nearby, with a transport truck just being admitted by the guard - rather heavily armed for corporate security. Despite not being that late, it's rather dark due to the rain-bearing clouds that have just started emptying their content. The windows of the office glow with invitingly warm light. A receptionist can be seen working through the glass door.
+
+ [21:46] * Steve_Briggs approaches the building at a brisk jog, pulling up the collar of his jacket to block as much of the rain as possible. Stopping just short of the door, he holds back and awaits Renala. "Well, hopefully this is the universe's way of getting our bad luck out of the way early." he says, glancing up at the sky. "Good things to come!" he adds with a smile.
+
+
+ [21:46] * Steve_Briggs approaches the building at a brisk jog, pulling up the collar of his jacket to block as much of the rain as possible. Stopping just short of the door, he holds back and awaits Renala. "Well, hopefully this is the universe's way of getting our bad luck out of the way early." he says, glancing up at the sky. "Good things to come!" he adds with a smile.
+
+ --> Stuff and things
+
+ <-- Stuff and things
+
+ [21:46] * Steve_Briggs approaches the building at a brisk jog, pulling up the collar of his jacket to block as much of the rain as possible. Stopping just short of the door, he holds back and awaits Renala. "Well, hopefully this is the universe's way of getting our bad luck out of the way early." he says, glancing up at the sky. "Good things to come!" he adds with a smile.
+ `
+
+var testLogPosts = [][]models.Post{
+	{
+		{
+			ID:       "UNASSIGNED",
+			LogID:    "UNASSIGNED",
+			Time:     parseDate(nil, "2013-07-25 21:28:00"),
+			Kind:     "action",
+			Nick:     "Tyranniac",
+			Text:     "| The Hnipinn Minerals local administrative center is a rather small building located next to the refinery. The area is mostly asphalt and industrial surroundings. The gate in the fence surrounding the refinery is nearby, with a transport truck just being admitted by the guard - rather heavily armed for corporate security. Despite not being that late, it's rather dark due to the rain-bearing clouds that have just started emptying their content. The windows of the office glow with invitingly warm light. A receptionist can be seen working through the glass door.",
+			Position: 1,
+		},
+		{
+			ID:       "UNASSIGNED",
+			LogID:    "UNASSIGNED",
+			Time:     parseDate(nil, "2013-07-25 21:46:00"),
+			Kind:     "action",
+			Nick:     "Steve_Briggs",
+			Text:     `approaches the building at a brisk jog, pulling up the collar of his jacket to block as much of the rain as possible. Stopping just short of the door, he holds back and awaits Renala. "Well, hopefully this is the universe's way of getting our bad luck out of the way early." he says, glancing up at the sky. "Good things to come!" he adds with a smile.`,
+			Position: 2,
+		},
+	},
+	{
+		{
+			ID:       "UNASSIGNED",
+			LogID:    "UNASSIGNED",
+			Time:     parseDate(nil, "2013-07-26 21:46:00"),
+			Kind:     "action",
+			Nick:     "Steve_Briggs",
+			Text:     `approaches the building at a brisk jog, pulling up the collar of his jacket to block as much of the rain as possible. Stopping just short of the door, he holds back and awaits Renala. "Well, hopefully this is the universe's way of getting our bad luck out of the way early." he says, glancing up at the sky. "Good things to come!" he adds with a smile.`,
+			Position: 1,
+		},
+		{
+			ID:       "UNASSIGNED",
+			LogID:    "UNASSIGNED",
+			Time:     parseDate(nil, "2013-07-26 21:46:00"),
+			Kind:     "action",
+			Nick:     "Steve_Briggs",
+			Text:     `approaches the building at a brisk jog, pulling up the collar of his jacket to block as much of the rain as possible. Stopping just short of the door, he holds back and awaits Renala. "Well, hopefully this is the universe's way of getting our bad luck out of the way early." he says, glancing up at the sky. "Good things to come!" he adds with a smile.`,
+			Position: 2,
+		},
+	},
+}
+
+var brokenLogNoPosts = `
+Date:
+September 27, 2014
+`
+
+var brokenLogBrokenPost = `
+Date:
+September 10, 2038
+
+[12:38] * Some_Character is lasered by a reaper, and turns to ash before even knowing what happened.
+
+[12:4 * Some_Other_Character also meets the same fate, because the Reapers are no joke.
+`
+
+var brokenLogBrokenDate = `
+Date:
+10 September, 2038
+
+[12:34] * =Scene= Stuff happens.
+`
+
+func parseDate(t *testing.T, date string) time.Time {
+	result, err := time.Parse("2006-01-02 15:04:05", date)
+	if err != nil {
+		if t != nil {
+			t.Fatal("Could not parse date", date, err)
+		} else {
+			panic("Could not parse date: " + err.Error())
+		}
+	}
+
+	return result
+}

internal/importers/forumlog/metadata.go

@@ -0,0 +1,35 @@
+package forumlog
+
+import (
+	"bufio"
+	"strings"
+)
+
+// ParseMetadata parses metadata, discards the broken parts, and returns the
+// parsed data as a map (`m`) and the position of the first IRC post (`n`)
+func ParseMetadata(data string) map[string][]string {
+	result := make(map[string][]string)
+	key := ""
+
+	scanner := bufio.NewScanner(strings.NewReader(data))
+	for scanner.Scan() {
+		line := strings.Trim(scanner.Text(), "\r\t  ")
+
+		if strings.HasPrefix(line, "[") {
+			break
+		}
+		if len(line) < 1 {
+			key = ""
+			continue
+		}
+
+		if key == "" {
+			split := strings.Split(line, ":")
+			key = split[0]
+		} else {
+			result[key] = append(result[key], line)
+		}
+	}
+
+	return result
+}

internal/importers/forumlog/metadata_test.go

@@ -0,0 +1,81 @@
+package forumlog_test
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"git.aiterp.net/rpdata/api/internal/importers/forumlog"
+)
+
+func TestParseMetadata(t *testing.T) {
+	data := forumlog.ParseMetadata(testData1)
+
+	assert.Equal(t, data["Date"], []string{
+		"August 11, 2014",
+		"August 12, 2014",
+		"August 13, 2014",
+	})
+	assert.Equal(t, data["Location"], []string{
+		"Aroste, an island off the Derraian coastline.",
+	})
+	assert.Equal(t, data["GM"], []string{
+		"Gisle",
+	})
+	assert.Equal(t, data["Length"], []string{
+		"847 posts",
+	})
+	assert.Equal(t, len(data["Stuff"]), 0)
+
+	for key := range data {
+		if strings.HasPrefix(key, "[") {
+			t.Error("It should have stopped at the IRC post, yet this key exists:", key)
+		}
+	}
+}
+
+var testData1 = `
+Date: August 2185
+August 11, 2014
+August 12, 2014
+August 13, 2014
+
+Location:
+Aroste, an island off the Derraian coastline.
+
+GM:
+Gisle
+
+Length:
+847 posts
+
+Characters:
+Calyx Vadris - Osiris
+Damien Monroe - Bowe
+Jason Wolfe - Dante
+Renala T'Iavay - Gisle
+Victoria Steels - MCB280
+
+NPCs:
+Recurring:
+Halisi - Tyranniac
+Jattic - Dante
+Jelvan Darennon - Gisle
+Leah - Dante
+Marissa T'Evin - Gisle
+Philip Lacour - Tyranniac
+
+One-off:
+Reidas Falten (Turian - Guard meeting them on their trip to the building)
+Sarian T'Dera (Asari - Asari opening fire on Calyx, Leah and Renala)
+Mariam Adams (Human - The receptionist.)
+Prerix Falten (Turian2 - Victoria's chair)
+Jonathan Lyng (OtherHuman - The one opening fire on Leah entering the garage)
+Alerena Teris (Teris - Asari met in the maintenance corridor)
+
+<-- Redorck Agency - August 11, 2014
+
+[13:19] * Leah looks over at Damien. "Haven't seen you around the agency." she says curiously, "New hire?"
+
+`

internal/importers/mirclike/log.go

@@ -0,0 +1,48 @@
+package mirclike
+
+import (
+	"errors"
+	"strings"
+	"time"
+
+	"git.aiterp.net/rpdata/api/models"
+)
+
+// ErrEmptyLog is returned by ParseLog if there are no (valid) posts in the log.
+var ErrEmptyLog = errors.New("No valid posts found in log")
+
+// ParseLog parses the log and returns the things that can be gleamed from them.
+func ParseLog(data string, date time.Time, strict bool) (models.Log, []models.Post, error) {
+	lines := strings.Split(data, "\n")
+	posts := make([]models.Post, 0, len(lines))
+	prev := models.Post{}
+
+	for _, line := range lines {
+		line = strings.Trim(line, "\r\t  ")
+		if len(line) < 1 {
+			continue
+		}
+
+		post, err := ParsePost(line, date, prev)
+		if err != nil {
+			if strict {
+				return models.Log{}, nil, err
+			}
+
+			continue
+		}
+
+		posts = append(posts, post)
+		prev = post
+	}
+
+	if len(posts) == 0 {
+		return models.Log{}, nil, ErrEmptyLog
+	}
+
+	log := models.Log{
+		Date: posts[0].Time,
+	}
+
+	return log, posts, nil
+}

internal/importers/mirclike/log_test.go

@@ -0,0 +1,69 @@
+package mirclike_test
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	"git.aiterp.net/rpdata/api/internal/importers/mirclike"
+	"git.aiterp.net/rpdata/api/models"
+)
+
+var testLog = strings.Join([]string{
+	"[11:21] * Va`ynna_Atana returns to the apartment at the end of another long day. She hangs up her jacket and a green scarf next to the door before going straight to the bathroom; she leaves the door open, though. She walked home with Uvena this time, but the two parted ways downstairs.",
+	"",
+	"[11:21] <=Scene=> The weather outside is not pleasant in the slightest. The storm is still going on, visibility is reduced and the worst gusts of wind can be felt inside the apartment as a faint shudder. ",
+	"",
+	"[11:",
+	"  [11:27] <Test> Stuff and things.",
+}, "\r\n")
+
+var testLogPosts = []models.Post{
+	{
+		ID:       "UNASSIGNED",
+		LogID:    "UNASSIGNED",
+		Time:     parseDate(nil, "2018-05-11 11:21:00"),
+		Kind:     "action",
+		Nick:     "Va`ynna_Atana",
+		Position: 1,
+		Text:     "returns to the apartment at the end of another long day. She hangs up her jacket and a green scarf next to the door before going straight to the bathroom; she leaves the door open, though. She walked home with Uvena this time, but the two parted ways downstairs.",
+	},
+	{
+		ID:       "UNASSIGNED",
+		LogID:    "UNASSIGNED",
+		Time:     parseDate(nil, "2018-05-11 11:21:00"),
+		Kind:     "scene",
+		Nick:     "=Scene=",
+		Position: 2,
+		Text:     "The weather outside is not pleasant in the slightest. The storm is still going on, visibility is reduced and the worst gusts of wind can be felt inside the apartment as a faint shudder.",
+	},
+	{
+		ID:       "UNASSIGNED",
+		LogID:    "UNASSIGNED",
+		Time:     parseDate(nil, "2018-05-11 11:27:00"),
+		Kind:     "text",
+		Nick:     "Test",
+		Position: 3,
+		Text:     "Stuff and things.",
+	},
+}
+
+func TestParseLog(t *testing.T) {
+	log, posts, err := mirclike.ParseLog(testLog, parseDate(t, "2018-05-11 00:00:00"), false)
+	if err != nil {
+		t.Fatal("ParseLog", err)
+	}
+
+	assert.Equal(t, testLogPosts, posts)
+	assert.Equal(t, posts[0].Time, log.Date, "Log's date should be the first post's.")
+}
+
+func TestParseLogErrors(t *testing.T) {
+	_, _, err1 := mirclike.ParseLog("\n\n\n\n\n   \n\t\r\n", time.Time{}, false)
+	_, _, err2 := mirclike.ParseLog("\n\n\n\n\n[14:57]* Stuff   \n\t\r\n", time.Time{}, true)
+
+	assert.Equal(t, mirclike.ErrEmptyLog, err1)
+	assert.Equal(t, mirclike.ErrNotPost, err2)
+}

internal/importers/mirclike/post.go

@@ -0,0 +1,92 @@
+package mirclike
+
+import (
+	"errors"
+	"strconv"
+	"strings"
+	"time"
+
+	"git.aiterp.net/rpdata/api/models"
+)
+
+// ErrNotPost is returned by parsePost if the line is empty or not a post.
+var ErrNotPost = errors.New("not a post")
+
+// ParsePost parses a post from a mirc-like line. If the previous post is included (it can be empty), it will be used
+// to determine whether midnight has passed.
+func ParsePost(line string, date time.Time, prev models.Post) (models.Post, error) {
+	// Do basic validation
+	line = strings.Trim(line, "  \t\n\r")
+	if len(line) == 0 || !strings.HasPrefix(line, "[") {
+		return models.Post{}, ErrNotPost
+	}
+
+	// Parse timestamp
+	tsEndIndex := strings.IndexByte(line, ']')
+	if tsEndIndex == -1 || len(line) < tsEndIndex+5 {
+		return models.Post{}, ErrNotPost
+	}
+	tsStr := line[1:tsEndIndex]
+	tsSplit := strings.Split(tsStr, ":")
+	tsUnits := make([]int, len(tsSplit))
+	if len(tsSplit) < 2 {
+		return models.Post{}, ErrNotPost
+	}
+	for i := range tsSplit {
+		n, err := strconv.Atoi(tsSplit[i])
+		if err != nil {
+			return models.Post{}, ErrNotPost
+		}
+
+		tsUnits[i] = n
+	}
+	if len(tsUnits) == 2 {
+		tsUnits = append(tsUnits, 0)
+	}
+
+	// Determine timestamp from parsed data and previous post.
+	ts := time.Date(date.Year(), date.Month(), date.Day(), tsUnits[0], tsUnits[1], tsUnits[2], 0, date.Location())
+	if !prev.Time.IsZero() && prev.Time.Sub(ts) > 30*time.Minute {
+		ts = time.Date(prev.Time.Year(), prev.Time.Month(), prev.Time.Day()+1, tsUnits[0], tsUnits[1], tsUnits[2], 0, date.Location())
+	}
+
+	if line[tsEndIndex+2] == '*' {
+		split := strings.SplitN(line[tsEndIndex+4:], " ", 2)
+
+		post := models.Post{
+			ID:       "UNASSIGNED",
+			LogID:    "UNASSIGNED",
+			Time:     ts,
+			Kind:     "action",
+			Nick:     strings.TrimLeft(split[0], "+@!~"),
+			Text:     split[1],
+			Position: prev.Position + 1,
+		}
+
+		if post.Nick[0] == '=' {
+			post.Kind = "scene"
+		}
+
+		return post, nil
+	} else if line[tsEndIndex+2] == '<' {
+		split := strings.SplitN(line[tsEndIndex+2:], " ", 2)
+
+		post := models.Post{
+			ID:       "UNASSIGNED",
+			LogID:    "UNASSIGNED",
+			Time:     ts,
+			Kind:     "text",
+			Nick:     strings.TrimLeft(split[0][1:len(split[0])-1], "+@!~"),
+			Text:     split[1],
+			Position: prev.Position + 1,
+		}
+
+		if post.Nick[0] == '=' {
+			post.Kind = "scene"
+		}
+
+		return post, nil
+	} else {
+		return models.Post{}, ErrNotPost
+	}
+}

internal/importers/mirclike/post_test.go

@@ -0,0 +1,126 @@
+package mirclike_test
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	"git.aiterp.net/rpdata/api/internal/importers/mirclike"
+	"git.aiterp.net/rpdata/api/models"
+)
+
+func TestParsePost(t *testing.T) {
+	table := []struct {
+		Input string
+		TS    string
+		Kind  string
+		Nick  string
+		Text  string
+	}{
+		{
+			"[12:34] * Stuff does things.",
+			"12:34:00", "action", "Stuff", "does things.",
+		},
+		{
+			"[12:34] <Stuff> Things said.",
+			"12:34:00", "text", "Stuff", "Things said.",
+		},
+		{
+			"[13:36:59] <Stuff> Things said.",
+			"13:36:59", "text", "Stuff", "Things said.",
+		},
+		{
+			"[23:59] <=Scene=> Scenery and such.",
+			"23:59:00", "scene", "=Scene=", "Scenery and such.",
+		},
+		{
+			"[01:10:11] * =Scene= Scenery and such from the forum or mIRC using my old script.",
+			"01:10:11", "scene", "=Scene=", "Scenery and such from the forum or mIRC using my old script.",
+		},
+	}
+
+	for i, row := range table {
+		t.Run(fmt.Sprintf("Row_%d", i), func(t *testing.T) {
+			post, err := mirclike.ParsePost(row.Input, time.Now(), models.Post{})
+			if err != nil {
+				t.Fatal("Could not parse post:", err)
+			}
+
+			assert.Equal(t, row.TS, post.Time.Format("15:04:05"), "Timestamps should match.")
+			assert.Equal(t, row.Kind, post.Kind, "Kinds should match.")
+			assert.Equal(t, row.Nick, post.Nick, "Kinds should match.")
+			assert.Equal(t, row.Text, post.Text, "Kinds should match.")
+		})
+	}
+}
+
+func TestParsePostErrors(t *testing.T) {
+	table := []struct {
+		Input string
+		Err   error
+	}{
+		{"[12:34] <Stuff> Things said.", nil},
+		{"[12:34] >Stuff> Things said.", mirclike.ErrNotPost},
+		{"12:34] <Stuff> Things said.", mirclike.ErrNotPost},
+		{"* Stuff Things said.", mirclike.ErrNotPost},
+		{"", mirclike.ErrNotPost},
+		{"[12:34 <Stuff> Things said.", mirclike.ErrNotPost},
+		{"[TE:XT] <Stuff> Things said.", mirclike.ErrNotPost},
+		{"[10] <Stuff> Things said.", mirclike.ErrNotPost},
+		{"[12:34:56:789] <Stuff> Things said.", nil},
+		{"[12:34:56.789] <Stuff> Things said.", mirclike.ErrNotPost},
+	}
+
+	for i, row := range table {
+		t.Run(fmt.Sprintf("Row_%d", i), func(t *testing.T) {
+			_, err := mirclike.ParsePost(row.Input, time.Now(), models.Post{})
+
+			assert.Equal(t, row.Err, err, "Error should match")
+		})
+	}
+}
+
+func TestParseNextDay(t *testing.T) {
+	table := []struct {
+		Prev time.Time
+		TS   string
+		Time time.Time
+	}{
+		{Prev: parseDate(t, "2019-01-12 12:00:00"), TS: "12:01", Time: parseDate(t, "2019-01-12 12:01:00")},
+		{Prev: parseDate(t, "2019-01-12 12:00:00"), TS: "11:53:13", Time: parseDate(t, "2019-01-12 11:53:13")},
+		{Prev: parseDate(t, "2019-04-08 23:51:59"), TS: "00:09", Time: parseDate(t, "2019-04-09 00:09:00")},
+		{Prev: parseDate(t, "2019-01-12 12:00:00"), TS: "11:29:59", Time: parseDate(t, "2019-01-13 11:29:59")},
+	}
+
+	for i, row := range table {
+		t.Run(fmt.Sprintf("Row_%d", i), func(t *testing.T) {
+			input := fmt.Sprintf("[%s] * Stuff does things.", row.TS)
+
+			post, err := mirclike.ParsePost(input, row.Prev, models.Post{Time: row.Prev})
+			if err != nil {
+				t.Fatal("Could not parse post:", err)
+			}
+
+			assert.Equal(t, row.Time, post.Time)
+		})
+	}
+}
+
+func parseDate(t *testing.T, date string) time.Time {
+	result, err := time.Parse("2006-01-02 15:04:05", date)
+	if err != nil {
+		if t != nil {
+			t.Fatal("Could not parse date", date, err)
+		} else {
+			panic("Could not parse date: " + err.Error())
+		}
+	}
+
+	return result
+}
+
+func formatDate(date time.Time) string {
+	return date.UTC().Format("2006-01-02 15:04:05")
+}

models/changes/db.go

@@ -16,7 +16,7 @@ var submitMutex sync.Mutex
 
 func list(query bson.M, limit int) ([]models.Change, error) {
 	changes := make([]models.Change, 0, 64)
-	err := collection.Find(query).Limit(limit).Sort("date").All(&changes)
+	err := collection.Find(query).Limit(limit).Sort("-date").All(&changes)
 
 	return changes, err
 }

models/changes/submit.go

@@ -1,6 +1,7 @@
 package changes
 
 import (
+	"log"
 	"strconv"
 	"time"
 
@@ -80,10 +81,12 @@ func Submit(model, op, author string, listed bool, keys []models.ChangeKey, obje
 			change.Chapters = append(change.Chapters, object)
 		case []models.Chapter:
 			change.Chapters = append(change.Chapters, object...)
+		default:
+			log.Printf("Warning: unrecognized object in change: %#+v", object)
 		}
 	}
 
-	// TODO: Push to subscribers
+	pushToSubscribers(change)
 
 	err = collection.Insert(&change)
 	if err != nil {

models/changes/subscribe.go

@@ -0,0 +1,88 @@
+package changes
+
+import (
+	"context"
+	"sync"
+
+	"git.aiterp.net/rpdata/api/models"
+)
+
+var subMutex sync.Mutex
+var subList []*subscription
+
+type subscription struct {
+	Keys     map[string]bool
+	Channel  chan<- models.Change
+	WildCard bool
+}
+
+// Subscribe subscribes to all changes.
+func Subscribe(ctx context.Context, keys []models.ChangeKey, wildcard bool) <-chan models.Change {
+	channel := make(chan models.Change, 64)
+	sub := &subscription{
+		Keys:     make(map[string]bool, len(keys)),
+		Channel:  channel,
+		WildCard: wildcard,
+	}
+
+	for _, key := range keys {
+		sub.Keys[mapKey(key)] = true
+	}
+
+	subMutex.Lock()
+	subList = append(subList, sub)
+	subMutex.Unlock()
+
+	go func() {
+		<-ctx.Done()
+
+		subMutex.Lock()
+		for i := range subList {
+			if subList[i] == sub {
+				subList = append(subList[:i], subList[i+1:]...)
+				break
+			}
+		}
+		subMutex.Unlock()
+	}()
+
+	return channel
+}
+
+func pushToSubscribers(change models.Change) {
+	keys := make([]string, len(change.Keys))
+	for i := range change.Keys {
+		keys[i] = mapKey(change.Keys[i])
+	}
+
+	subMutex.Lock()
+SubLoop:
+	for _, sub := range subList {
+		if sub.WildCard && change.Listed {
+			select {
+			case sub.Channel <- change:
+			default:
+			}
+		} else {
+			for _, key := range keys {
+				if sub.Keys[key] {
+					select {
+					case sub.Channel <- change:
+					default:
+					}
+
+					continue SubLoop
+				}
+			}
+		}
+	}
+	subMutex.Unlock()
+}
+
+func mapKey(ck models.ChangeKey) string {
+	return ck.Model.String() + "." + ck.ID
+}
+
+func init() {
+	subList = make([]*subscription, 0, 16)
+}

models/log-importer.go

@@ -0,0 +1,49 @@
+package models
+
+import (
+	"fmt"
+	"io"
+	"strconv"
+)
+
+// LogImporter describes a model related log importing.
+type LogImporter string
+
+const (
+	// LogImporterMircLike is a value of LogImporter
+	LogImporterMircLike LogImporter = "MircLike"
+	// LogImporterForumLog is a value of LogImporter
+	LogImporterForumLog LogImporter = "ForumLog"
+)
+
+// IsValid returns true if the underlying string is one of the correct values.
+func (e LogImporter) IsValid() bool {
+	switch e {
+	case LogImporterForumLog, LogImporterMircLike:
+		return true
+	}
+	return false
+}
+
+func (e LogImporter) String() string {
+	return string(e)
+}
+
+// UnmarshalGQL unmarshals the underlying graphql value.
+func (e *LogImporter) UnmarshalGQL(v interface{}) error {
+	str, ok := v.(string)
+	if !ok {
+		return fmt.Errorf("enums must be strings")
+	}
+
+	*e = LogImporter(str)
+	if !e.IsValid() {
+		return fmt.Errorf("%s is not a valid LogImporter", str)
+	}
+	return nil
+}
+
+// MarshalGQL marshals the underlying graphql value.
+func (e LogImporter) MarshalGQL(w io.Writer) {
+	fmt.Fprint(w, strconv.Quote(e.String()))
+}

models/logs/import.go

@@ -0,0 +1,93 @@
+package logs
+
+import (
+	"errors"
+	"time"
+
+	"git.aiterp.net/rpdata/api/internal/importers/mirclike"
+
+	"git.aiterp.net/rpdata/api/models/posts"
+
+	"git.aiterp.net/rpdata/api/models/channels"
+
+	"git.aiterp.net/rpdata/api/internal/importers/forumlog"
+
+	"git.aiterp.net/rpdata/api/models"
+)
+
+// An ImportedLog contains data about an imported log.
+type ImportedLog struct {
+	Log   models.Log
+	Posts []models.Post
+}
+
+// Import makes a log and posts object from different formats.
+func Import(importer models.LogImporter, date time.Time, tz *time.Location, channelName string, data string) ([]ImportedLog, error) {
+	results := make([]ImportedLog, 0, 8)
+
+	eventName := ""
+	if channel, err := channels.FindName(channelName); err != nil {
+		eventName = channel.EventName
+	}
+
+	date = date.In(tz)
+
+	switch importer {
+	case models.LogImporterMircLike:
+		{
+			if date.IsZero() {
+				return nil, errors.New("Date is not optional for mirc-like logs")
+			}
+
+			parsedLog, parsedPosts, err := mirclike.ParseLog(data, date, true)
+			if err != nil {
+				return nil, err
+			}
+
+			log, err := Add(parsedLog.Date, channelName, "", eventName, "", false)
+			if err != nil {
+				return nil, err
+			}
+
+			posts, err := posts.AddMany(log, parsedPosts)
+			if err != nil {
+				return nil, err
+			}
+
+			results = append(results, ImportedLog{
+				Log:   log,
+				Posts: posts,
+			})
+		}
+	case models.LogImporterForumLog:
+		{
+			parseResults, err := forumlog.ParseLogs(data, tz)
+			if err != nil {
+				return nil, err
+			}
+
+			for _, result := range parseResults {
+				log, err := Add(result.Log.Date, channelName, "", eventName, "", false)
+				if err != nil {
+					return nil, err
+				}
+
+				posts, err := posts.AddMany(log, result.Posts)
+				if err != nil {
+					return nil, err
+				}
+
+				results = append(results, ImportedLog{
+					Log:   log,
+					Posts: posts,
+				})
+			}
+		}
+	default:
+		{
+			return nil, errors.New("Invalid importer: " + importer.String())
+		}
+	}
+
+	return results, nil
+}

models/logs/update-characters.go

@@ -1,6 +1,7 @@
 package logs
 
 import (
+	"errors"
 	"strings"
 	"time"
 
@@ -8,23 +9,25 @@ import (
 	"git.aiterp.net/rpdata/api/models"
 	"git.aiterp.net/rpdata/api/models/characters"
 	"git.aiterp.net/rpdata/api/models/posts"
+	"git.aiterp.net/rpdata/api/models/unknownnicks"
 	"github.com/globalsign/mgo/bson"
 )
 
 var updateTask = task.New(time.Second*60, RunFullUpdate)
 
 // UpdateCharacters updates the characters for the given log.
-func UpdateCharacters(log models.Log) (models.Log, error) {
+func UpdateCharacters(log models.Log, unknowns map[string]int) (models.Log, error) {
 	posts, err := posts.List(&posts.Filter{LogID: &log.ShortID, Kind: []string{"action", "text", "chars"}, Limit: 0})
 	if err != nil {
 		return models.Log{}, err
 	}
 
+	counts := make(map[string]int)
 	added := make(map[string]bool)
 	removed := make(map[string]bool)
 	for _, post := range posts {
 		if post.Kind == "text" || post.Kind == "action" {
-			if strings.HasPrefix(post.Text, "(") || strings.Contains(post.Nick, "(") || strings.Contains(post.Nick, "[E]") {
+			if strings.HasPrefix(post.Text, "(") || strings.Contains(post.Nick, "(") || strings.Contains(post.Nick, "[E]") || strings.HasSuffix(post.Nick, "|") {
 				continue
 			}
 
@@ -36,6 +39,7 @@ func UpdateCharacters(log models.Log) (models.Log, error) {
 			}
 
 			added[post.Nick] = true
+			counts[post.Nick]++
 		}
 		if post.Kind == "chars" {
 			tokens := strings.Fields(post.Text)
@@ -63,6 +67,10 @@ func UpdateCharacters(log models.Log) (models.Log, error) {
 
 	characterIDs := make([]string, len(characters))
 	for i, char := range characters {
+		if char.Name == "(Hidden)" {
+			continue
+		}
+
 		characterIDs[i] = char.ID
 	}
 
@@ -73,6 +81,23 @@ func UpdateCharacters(log models.Log) (models.Log, error) {
 
 	log.CharacterIDs = characterIDs
 
+	if len(nicks) > 0 && unknowns != nil {
+	NickLoop:
+		for nick := range added {
+			if !added[nick] {
+				continue
+			}
+
+			for _, character := range characters {
+				if character.HasNick(nick) {
+					continue NickLoop
+				}
+			}
+
+			unknowns[nick] += counts[nick]
+		}
+	}
+
 	return log, nil
 }
 
@@ -84,9 +109,11 @@ func RunFullUpdate() error {
 		return err
 	}
 
+	unknowns := make(map[string]int, 256)
+
 	log := models.Log{}
 	for iter.Next(&log) {
-		_, err = UpdateCharacters(log)
+		_, err = UpdateCharacters(log, unknowns)
 		if err != nil {
 			return err
 		}
@@ -97,6 +124,11 @@ func RunFullUpdate() error {
 		return err
 	}
 
+	err = unknownnicks.Update(unknowns)
+	if err != nil {
+		return errors.New("Failed to commit unknown nicks update: " + err.Error())
+	}
+
 	return nil
 }
 

models/posts/add-many.go

@@ -0,0 +1,36 @@
+package posts
+
+import (
+	"git.aiterp.net/rpdata/api/internal/counter"
+	"git.aiterp.net/rpdata/api/models"
+)
+
+// AddMany adds multiple posts in on query. Each post gets a new ID and is associated with the log.
+func AddMany(log models.Log, posts []models.Post) ([]models.Post, error) {
+	docs := make([]interface{}, len(posts))
+	copies := make([]models.Post, len(posts))
+
+	mutex.RLock()
+	defer mutex.RUnlock()
+
+	startPosition, err := counter.NextMany("next_post_id", log.ShortID, len(posts))
+	if err != nil {
+		return nil, err
+	}
+
+	for i, post := range posts {
+		post.ID = generateID(post.Time)
+		post.LogID = log.ShortID
+		post.Position = startPosition + i
+
+		docs[i] = post
+		copies[i] = post
+	}
+
+	err = collection.Insert(docs...)
+	if err != nil {
+		return nil, err
+	}
+
+	return copies, nil
+}

models/posts/remove.go

@@ -1,6 +1,7 @@
 package posts
 
 import (
+	"git.aiterp.net/rpdata/api/internal/counter"
 	"git.aiterp.net/rpdata/api/models"
 	"github.com/globalsign/mgo/bson"
 )
@@ -20,6 +21,8 @@ func Remove(post models.Post) (models.Post, error) {
 		return models.Post{}, err
 	}
 
+	counter.NextMany("next_post_id", post.LogID, -1)
+
 	return post, nil
 }
 

models/unknown-nick.go

@@ -0,0 +1,7 @@
+package models
+
+// UnknownNick represents an unknown nick name.
+type UnknownNick struct {
+	Nick  string `bson:"_id"`
+	Score int    `bson:"score"`
+}

models/unknownnicks/db.go

@@ -0,0 +1,37 @@
+package unknownnicks
+
+import (
+	"log"
+
+	"git.aiterp.net/rpdata/api/internal/store"
+	"git.aiterp.net/rpdata/api/models"
+	"github.com/globalsign/mgo"
+)
+
+var collection *mgo.Collection
+
+func list(query interface{}, limit int) ([]models.UnknownNick, error) {
+	size := 64
+	if limit > 0 {
+		size = limit
+	}
+	posts := make([]models.UnknownNick, 0, size)
+
+	err := collection.Find(query).Limit(limit).Sort("-score").All(&posts)
+	if err != nil {
+		return nil, err
+	}
+
+	return posts, nil
+}
+
+func init() {
+	store.HandleInit(func(db *mgo.Database) {
+		collection = db.C("logbot3.unkownnicks")
+
+		err := collection.EnsureIndexKey("weight")
+		if err != nil {
+			log.Fatalln("init logbot3.unkownnicks:", err)
+		}
+	})
+}

models/unknownnicks/list.go

@@ -0,0 +1,10 @@
+package unknownnicks
+
+import (
+	"git.aiterp.net/rpdata/api/models"
+)
+
+// List lists the top N unknown nicks.
+func List(limit int) ([]models.UnknownNick, error) {
+	return list(nil, limit)
+}

models/unknownnicks/remove.go

@@ -0,0 +1 @@
+package unknownnicks

models/unknownnicks/update.go

@@ -0,0 +1,22 @@
+package unknownnicks
+
+import (
+	"github.com/globalsign/mgo/bson"
+)
+
+// Update updates the map, replacing it with the scores provided here.
+func Update(updateMap map[string]int) error {
+	_, err := collection.RemoveAll(bson.M{})
+	if err != nil {
+		return err
+	}
+
+	for nick, score := range updateMap {
+		_, err := collection.UpsertId(nick, bson.M{"$set": bson.M{"score": score}})
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}