Browse Source

graph2: Added importLog query and associated models and repository functions.

module-madness-pointers
Gisle Aune 6 years ago
parent
commit
a58535fcb9
  1. 2
      graph2/gqlgen.yml
  2. 45
      graph2/queries/log.go
  3. 3
      graph2/schema/root.gql
  4. 32
      graph2/schema/types/Log.gql
  5. 18
      internal/counter/counter.go
  6. 4
      internal/importers/forumlog/logs.go
  7. 8
      internal/importers/forumlog/logs_test.go
  8. 49
      models/log-importer.go
  9. 93
      models/logs/import.go
  10. 34
      models/posts/add-many.go
  11. BIN
      test.prof

2
graph2/gqlgen.yml

@ -31,6 +31,8 @@ models:
model: git.aiterp.net/rpdata/api/models.Log
LogsFilter:
model: git.aiterp.net/rpdata/api/models/logs.Filter
LogImporter:
model: git.aiterp.net/rpdata/api/models.LogImporter
Chapter:
model: git.aiterp.net/rpdata/api/models.Chapter
fields:

45
graph2/queries/log.go

@ -4,6 +4,7 @@ import (
"context"
"errors"
"strings"
"time"
"git.aiterp.net/rpdata/api/models/channels"
@ -96,6 +97,50 @@ func (r *mutationResolver) AddLog(ctx context.Context, input input.LogAddInput)
return log, nil
}
func (r *mutationResolver) ImportLog(ctx context.Context, input input.LogImportInput) ([]models.Log, error) {
token := auth.TokenFromContext(ctx)
if !token.Authenticated() || !token.Permitted("log.add") {
return nil, errors.New("You are not permitted to add logs")
}
date := time.Time{}
if input.Date != nil {
date = *input.Date
}
tz := time.UTC
if input.Timezone != nil {
parsedTZ, err := time.LoadLocation(*input.Timezone)
if err != nil {
return nil, errors.New("Unknown timezone: " + *input.Timezone)
}
tz = parsedTZ
}
results, err := logs.Import(input.Importer, date, tz, input.ChannelName, input.Data)
if err != nil {
return nil, err
}
newLogs := make([]models.Log, 0, len(results))
for _, result := range results {
go func() {
changes.Submit("Log", "add", token.UserID, true, changekeys.Many(result.Log), result.Log)
changes.Submit("Post", "add", token.UserID, true, changekeys.Many(result.Log, result.Posts), result.Posts)
}()
log, err := logs.UpdateCharacters(result.Log)
if err != nil {
log = result.Log
}
newLogs = append(newLogs, log)
}
return newLogs, nil
}
func (r *mutationResolver) EditLog(ctx context.Context, input input.LogEditInput) (models.Log, error) {
token := auth.TokenFromContext(ctx)
if !token.Authenticated() || !token.Permitted("log.edit") {

3
graph2/schema/root.gql

@ -95,6 +95,9 @@ type Mutation {
# Add a new log
addLog(input: LogAddInput!): Log!
# Import a log
importLog(input: LogImportInput!): [Log!]!
# Edit a log
editLog(input: LogEditInput!): Log!

32
graph2/schema/types/Log.gql

@ -100,4 +100,36 @@ input LogEditInput {
input LogRemoveInput {
# The id of the log to remove
id: String!
}
"""
Input for importLog mutation.
"""
input LogImportInput {
"The channel of the log, alwyas required."
channelName: String!
"Which importer to use."
importer: LogImporter!
"The timezone of the import, all log and post dates will be treated as it. It will be UTC if none is specified."
timezone: String
"The date of the log, if not provided in the log body."
date: Date
"The log body itself."
data: String!
}
enum LogImporter {
"""
mIRC-like: This importer parses logs that copied from mIRC posts without spaces.
"""
MircLike
"""
Forum log: This importer parses the format on the forum. The displayed log, not the post source.
"""
ForumLog
}

18
internal/counter/counter.go

@ -30,6 +30,24 @@ func Next(category, name string) (int, error) {
return doc.Value, nil
}
// NextMany gets the next value of a counter, or an error if it hasn't, and increments by a specified value.
// Any value `returned` to `returned+(increment-1)` should then be safe to use.
func NextMany(category, name string, increment int) (int, error) {
id := category + "." + name
doc := counter{}
_, err := collection.Find(bson.M{"_id": id}).Apply(mgo.Change{
Update: bson.M{"$inc": bson.M{"value": 1}},
Upsert: true,
ReturnNew: true,
}, &doc)
if err != nil {
return -1, err
}
return doc.Value, nil
}
func init() {
store.HandleInit(func(db *mgo.Database) {
collection = db.C("core.counters")

4
internal/importers/forumlog/logs.go

@ -18,14 +18,14 @@ type ParsedLog struct {
}
// ParseLogs parses the logs from the data.
func ParseLogs(data string) ([]ParsedLog, error) {
func ParseLogs(data string, tz *time.Location) ([]ParsedLog, error) {
metadata := ParseMetadata(data)
results := make([]ParsedLog, 0, len(metadata["Date"]))
scanner := bufio.NewScanner(strings.NewReader(data))
for i, dateStr := range metadata["Date"] {
// Parse date
date, err := time.Parse("January 2, 2006", dateStr)
date, err := time.ParseInLocation("January 2, 2006", dateStr, tz)
if err != nil {
return nil, fmt.Errorf("Failed to parse date #%d: %#+v is not the in the correct format of \"January 2, 2006\"", i+1, dateStr)
}

8
internal/importers/forumlog/logs_test.go

@ -12,7 +12,7 @@ import (
)
func TestParseLogs(t *testing.T) {
results, err := forumlog.ParseLogs(testLog)
results, err := forumlog.ParseLogs(testLog, time.UTC)
if err != nil {
t.Fatalf("Parse: %s", err)
}
@ -23,9 +23,9 @@ func TestParseLogs(t *testing.T) {
}
func TestParseLogsErrors(t *testing.T) {
_, err1 := forumlog.ParseLogs(brokenLogNoPosts)
_, err2 := forumlog.ParseLogs(brokenLogBrokenPost)
_, err3 := forumlog.ParseLogs(brokenLogBrokenDate)
_, err1 := forumlog.ParseLogs(brokenLogNoPosts, time.UTC)
_, err2 := forumlog.ParseLogs(brokenLogBrokenPost, time.UTC)
_, err3 := forumlog.ParseLogs(brokenLogBrokenDate, time.UTC)
t.Log("Should be about no posts:", err1)
t.Log("Should be about a broken post:", err2)

49
models/log-importer.go

@ -0,0 +1,49 @@
package models
import (
"fmt"
"io"
"strconv"
)
// LogImporter describes a model related log importing.
type LogImporter string
const (
// LogImporterMircLike is a value of LogImporter
LogImporterMircLike LogImporter = "MircLike"
// LogImporterForumLog is a value of LogImporter
LogImporterForumLog LogImporter = "ForumLog"
)
// IsValid returns true if the underlying string is one of the correct values.
func (e LogImporter) IsValid() bool {
switch e {
case LogImporterForumLog, LogImporterMircLike:
return true
}
return false
}
func (e LogImporter) String() string {
return string(e)
}
// UnmarshalGQL unmarshals the underlying graphql value.
func (e *LogImporter) UnmarshalGQL(v interface{}) error {
str, ok := v.(string)
if !ok {
return fmt.Errorf("enums must be strings")
}
*e = LogImporter(str)
if !e.IsValid() {
return fmt.Errorf("%s is not a valid LogImporter", str)
}
return nil
}
// MarshalGQL marshals the underlying graphql value.
func (e LogImporter) MarshalGQL(w io.Writer) {
fmt.Fprint(w, strconv.Quote(e.String()))
}

93
models/logs/import.go

@ -0,0 +1,93 @@
package logs
import (
"errors"
"time"
"git.aiterp.net/rpdata/api/internal/importers/mirclike"
"git.aiterp.net/rpdata/api/models/posts"
"git.aiterp.net/rpdata/api/models/channels"
"git.aiterp.net/rpdata/api/internal/importers/forumlog"
"git.aiterp.net/rpdata/api/models"
)
// An ImportedLog contains data about an imported log.
type ImportedLog struct {
Log models.Log
Posts []models.Post
}
// Import makes a log and posts object from different formats.
func Import(importer models.LogImporter, date time.Time, tz *time.Location, channelName string, data string) ([]ImportedLog, error) {
results := make([]ImportedLog, 0, 8)
eventName := ""
if channel, err := channels.FindName(channelName); err != nil {
eventName = channel.EventName
}
date = date.In(tz)
switch importer {
case models.LogImporterMircLike:
{
if date.IsZero() {
return nil, errors.New("Date is not optional for mirc-like logs")
}
parsedLog, parsedPosts, err := mirclike.ParseLog(data, date, true)
if err != nil {
return nil, err
}
log, err := Add(parsedLog.Date, channelName, "", eventName, "", false)
if err != nil {
return nil, err
}
posts, err := posts.AddMany(log, parsedPosts)
if err != nil {
return nil, err
}
results = append(results, ImportedLog{
Log: log,
Posts: posts,
})
}
case models.LogImporterForumLog:
{
parseResults, err := forumlog.ParseLogs(data, tz)
if err != nil {
return nil, err
}
for _, result := range parseResults {
log, err := Add(result.Log.Date, channelName, "", eventName, "", false)
if err != nil {
return nil, err
}
posts, err := posts.AddMany(log, result.Posts)
if err != nil {
return nil, err
}
results = append(results, ImportedLog{
Log: log,
Posts: posts,
})
}
}
default:
{
return nil, errors.New("Invalid importer: " + importer.String())
}
}
return results, nil
}

34
models/posts/add-many.go

@ -0,0 +1,34 @@
package posts
import (
"git.aiterp.net/rpdata/api/internal/counter"
"git.aiterp.net/rpdata/api/models"
)
// AddMany adds multiple posts in on query. Each post gets a new ID and is associated with the log.
func AddMany(log models.Log, posts []models.Post) ([]models.Post, error) {
docs := make([]interface{}, len(posts))
mutex.RLock()
defer mutex.RUnlock()
startPosition, err := counter.NextMany("next_post_id", log.ShortID, len(posts))
if err != nil {
return nil, err
}
for i, post := range posts {
post.ID = generateID(post.Time)
post.LogID = log.ShortID
post.Position = startPosition + i
docs[i] = post
}
err = collection.Insert(docs...)
if err != nil {
return nil, err
}
return posts, nil
}

BIN
test.prof

Loading…
Cancel
Save