graph2: Added importLog query and associated models and repository functions.

graph2/gqlgen.yml

@@ -31,6 +31,8 @@ models:
     model: git.aiterp.net/rpdata/api/models.Log
   LogsFilter:
     model: git.aiterp.net/rpdata/api/models/logs.Filter
+  LogImporter:
+    model: git.aiterp.net/rpdata/api/models.LogImporter
   Chapter:
     model: git.aiterp.net/rpdata/api/models.Chapter
     fields:

graph2/queries/log.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"strings"
+	"time"
 
 	"git.aiterp.net/rpdata/api/models/channels"
 
@@ -96,6 +97,50 @@ func (r *mutationResolver) AddLog(ctx context.Context, input input.LogAddInput)
 	return log, nil
 }
 
+func (r *mutationResolver) ImportLog(ctx context.Context, input input.LogImportInput) ([]models.Log, error) {
+	token := auth.TokenFromContext(ctx)
+	if !token.Authenticated() || !token.Permitted("log.add") {
+		return nil, errors.New("You are not permitted to add logs")
+	}
+
+	date := time.Time{}
+	if input.Date != nil {
+		date = *input.Date
+	}
+
+	tz := time.UTC
+	if input.Timezone != nil {
+		parsedTZ, err := time.LoadLocation(*input.Timezone)
+		if err != nil {
+			return nil, errors.New("Unknown timezone: " + *input.Timezone)
+		}
+
+		tz = parsedTZ
+	}
+
+	results, err := logs.Import(input.Importer, date, tz, input.ChannelName, input.Data)
+	if err != nil {
+		return nil, err
+	}
+
+	newLogs := make([]models.Log, 0, len(results))
+	for _, result := range results {
+		go func() {
+			changes.Submit("Log", "add", token.UserID, true, changekeys.Many(result.Log), result.Log)
+			changes.Submit("Post", "add", token.UserID, true, changekeys.Many(result.Log, result.Posts), result.Posts)
+		}()
+
+		log, err := logs.UpdateCharacters(result.Log)
+		if err != nil {
+			log = result.Log
+		}
+
+		newLogs = append(newLogs, log)
+	}
+
+	return newLogs, nil
+}
+
 func (r *mutationResolver) EditLog(ctx context.Context, input input.LogEditInput) (models.Log, error) {
 	token := auth.TokenFromContext(ctx)
 	if !token.Authenticated() || !token.Permitted("log.edit") {

graph2/schema/root.gql

@@ -95,6 +95,9 @@ type Mutation {
   # Add a new log
   addLog(input: LogAddInput!): Log!
 
+  # Import a log
+  importLog(input: LogImportInput!): [Log!]!
+
   # Edit a log
   editLog(input: LogEditInput!): Log!
 

graph2/schema/types/Log.gql

@@ -100,4 +100,36 @@ input LogEditInput {
 input LogRemoveInput {
   # The id of the log to remove
   id: String!
+}
+
+"""
+Input for importLog mutation.
+"""
+input LogImportInput {
+  "The channel of the log, alwyas required."
+  channelName: String!
+
+  "Which importer to use."
+  importer: LogImporter!
+
+  "The timezone of the import, all log and post dates will be treated as it. It will be UTC if none is specified."
+  timezone: String
+
+  "The date of the log, if not provided in the log body."
+  date: Date
+
+  "The log body itself."
+  data: String!
+}
+
+enum LogImporter {
+  """
+  mIRC-like: This importer parses logs that copied from mIRC posts without spaces.
+  """
+  MircLike
+
+  """
+  Forum log: This importer parses the format on the forum. The displayed log, not the post source.
+  """
+  ForumLog
 }

internal/counter/counter.go

@@ -30,6 +30,24 @@ func Next(category, name string) (int, error) {
 	return doc.Value, nil
 }
 
+// NextMany gets the next value of a counter, or an error if it hasn't, and increments by a specified value.
+// Any value `returned` to `returned+(increment-1)` should then be safe to use.
+func NextMany(category, name string, increment int) (int, error) {
+	id := category + "." + name
+	doc := counter{}
+
+	_, err := collection.Find(bson.M{"_id": id}).Apply(mgo.Change{
+		Update:    bson.M{"$inc": bson.M{"value": 1}},
+		Upsert:    true,
+		ReturnNew: true,
+	}, &doc)
+	if err != nil {
+		return -1, err
+	}
+
+	return doc.Value, nil
+}
+
 func init() {
 	store.HandleInit(func(db *mgo.Database) {
 		collection = db.C("core.counters")

internal/importers/forumlog/logs.go

@@ -18,14 +18,14 @@ type ParsedLog struct {
 }
 
 // ParseLogs parses the logs from the data.
-func ParseLogs(data string) ([]ParsedLog, error) {
+func ParseLogs(data string, tz *time.Location) ([]ParsedLog, error) {
 	metadata := ParseMetadata(data)
 	results := make([]ParsedLog, 0, len(metadata["Date"]))
 	scanner := bufio.NewScanner(strings.NewReader(data))
 
 	for i, dateStr := range metadata["Date"] {
 		// Parse date
-		date, err := time.Parse("January 2, 2006", dateStr)
+		date, err := time.ParseInLocation("January 2, 2006", dateStr, tz)
 		if err != nil {
 			return nil, fmt.Errorf("Failed to parse date #%d: %#+v is not the in the correct format of \"January 2, 2006\"", i+1, dateStr)
 		}

internal/importers/forumlog/logs_test.go

@@ -12,7 +12,7 @@ import (
 )
 
 func TestParseLogs(t *testing.T) {
-	results, err := forumlog.ParseLogs(testLog)
+	results, err := forumlog.ParseLogs(testLog, time.UTC)
 	if err != nil {
 		t.Fatalf("Parse: %s", err)
 	}
@@ -23,9 +23,9 @@ func TestParseLogs(t *testing.T) {
 }
 
 func TestParseLogsErrors(t *testing.T) {
-	_, err1 := forumlog.ParseLogs(brokenLogNoPosts)
-	_, err2 := forumlog.ParseLogs(brokenLogBrokenPost)
-	_, err3 := forumlog.ParseLogs(brokenLogBrokenDate)
+	_, err1 := forumlog.ParseLogs(brokenLogNoPosts, time.UTC)
+	_, err2 := forumlog.ParseLogs(brokenLogBrokenPost, time.UTC)
+	_, err3 := forumlog.ParseLogs(brokenLogBrokenDate, time.UTC)
 
 	t.Log("Should be about no posts:", err1)
 	t.Log("Should be about a broken post:", err2)

models/log-importer.go

@@ -0,0 +1,49 @@
+package models
+
+import (
+	"fmt"
+	"io"
+	"strconv"
+)
+
+// LogImporter describes a model related log importing.
+type LogImporter string
+
+const (
+	// LogImporterMircLike is a value of LogImporter
+	LogImporterMircLike LogImporter = "MircLike"
+	// LogImporterForumLog is a value of LogImporter
+	LogImporterForumLog LogImporter = "ForumLog"
+)
+
+// IsValid returns true if the underlying string is one of the correct values.
+func (e LogImporter) IsValid() bool {
+	switch e {
+	case LogImporterForumLog, LogImporterMircLike:
+		return true
+	}
+	return false
+}
+
+func (e LogImporter) String() string {
+	return string(e)
+}
+
+// UnmarshalGQL unmarshals the underlying graphql value.
+func (e *LogImporter) UnmarshalGQL(v interface{}) error {
+	str, ok := v.(string)
+	if !ok {
+		return fmt.Errorf("enums must be strings")
+	}
+
+	*e = LogImporter(str)
+	if !e.IsValid() {
+		return fmt.Errorf("%s is not a valid LogImporter", str)
+	}
+	return nil
+}
+
+// MarshalGQL marshals the underlying graphql value.
+func (e LogImporter) MarshalGQL(w io.Writer) {
+	fmt.Fprint(w, strconv.Quote(e.String()))
+}

models/logs/import.go

@@ -0,0 +1,93 @@
+package logs
+
+import (
+	"errors"
+	"time"
+
+	"git.aiterp.net/rpdata/api/internal/importers/mirclike"
+
+	"git.aiterp.net/rpdata/api/models/posts"
+
+	"git.aiterp.net/rpdata/api/models/channels"
+
+	"git.aiterp.net/rpdata/api/internal/importers/forumlog"
+
+	"git.aiterp.net/rpdata/api/models"
+)
+
+// An ImportedLog contains data about an imported log.
+type ImportedLog struct {
+	Log   models.Log
+	Posts []models.Post
+}
+
+// Import makes a log and posts object from different formats.
+func Import(importer models.LogImporter, date time.Time, tz *time.Location, channelName string, data string) ([]ImportedLog, error) {
+	results := make([]ImportedLog, 0, 8)
+
+	eventName := ""
+	if channel, err := channels.FindName(channelName); err != nil {
+		eventName = channel.EventName
+	}
+
+	date = date.In(tz)
+
+	switch importer {
+	case models.LogImporterMircLike:
+		{
+			if date.IsZero() {
+				return nil, errors.New("Date is not optional for mirc-like logs")
+			}
+
+			parsedLog, parsedPosts, err := mirclike.ParseLog(data, date, true)
+			if err != nil {
+				return nil, err
+			}
+
+			log, err := Add(parsedLog.Date, channelName, "", eventName, "", false)
+			if err != nil {
+				return nil, err
+			}
+
+			posts, err := posts.AddMany(log, parsedPosts)
+			if err != nil {
+				return nil, err
+			}
+
+			results = append(results, ImportedLog{
+				Log:   log,
+				Posts: posts,
+			})
+		}
+	case models.LogImporterForumLog:
+		{
+			parseResults, err := forumlog.ParseLogs(data, tz)
+			if err != nil {
+				return nil, err
+			}
+
+			for _, result := range parseResults {
+				log, err := Add(result.Log.Date, channelName, "", eventName, "", false)
+				if err != nil {
+					return nil, err
+				}
+
+				posts, err := posts.AddMany(log, result.Posts)
+				if err != nil {
+					return nil, err
+				}
+
+				results = append(results, ImportedLog{
+					Log:   log,
+					Posts: posts,
+				})
+			}
+		}
+	default:
+		{
+			return nil, errors.New("Invalid importer: " + importer.String())
+		}
+	}
+
+	return results, nil
+}

models/posts/add-many.go

@@ -0,0 +1,34 @@
+package posts
+
+import (
+	"git.aiterp.net/rpdata/api/internal/counter"
+	"git.aiterp.net/rpdata/api/models"
+)
+
+// AddMany adds multiple posts in on query. Each post gets a new ID and is associated with the log.
+func AddMany(log models.Log, posts []models.Post) ([]models.Post, error) {
+	docs := make([]interface{}, len(posts))
+
+	mutex.RLock()
+	defer mutex.RUnlock()
+
+	startPosition, err := counter.NextMany("next_post_id", log.ShortID, len(posts))
+	if err != nil {
+		return nil, err
+	}
+
+	for i, post := range posts {
+		post.ID = generateID(post.Time)
+		post.LogID = log.ShortID
+		post.Position = startPosition + i
+
+		docs[i] = post
+	}
+
+	err = collection.Insert(docs...)
+	if err != nil {
+		return nil, err
+	}
+
+	return posts, nil
+}