diff --git a/graph2/gqlgen.yml b/graph2/gqlgen.yml index 820213e..ee85a21 100644 --- a/graph2/gqlgen.yml +++ b/graph2/gqlgen.yml @@ -31,6 +31,8 @@ models: model: git.aiterp.net/rpdata/api/models.Log LogsFilter: model: git.aiterp.net/rpdata/api/models/logs.Filter + LogImporter: + model: git.aiterp.net/rpdata/api/models.LogImporter Chapter: model: git.aiterp.net/rpdata/api/models.Chapter fields: diff --git a/graph2/queries/log.go b/graph2/queries/log.go index 9763a6d..12b1efc 100644 --- a/graph2/queries/log.go +++ b/graph2/queries/log.go @@ -4,6 +4,7 @@ import ( "context" "errors" "strings" + "time" "git.aiterp.net/rpdata/api/models/channels" @@ -96,6 +97,50 @@ func (r *mutationResolver) AddLog(ctx context.Context, input input.LogAddInput) return log, nil } +func (r *mutationResolver) ImportLog(ctx context.Context, input input.LogImportInput) ([]models.Log, error) { + token := auth.TokenFromContext(ctx) + if !token.Authenticated() || !token.Permitted("log.add") { + return nil, errors.New("You are not permitted to add logs") + } + + date := time.Time{} + if input.Date != nil { + date = *input.Date + } + + tz := time.UTC + if input.Timezone != nil { + parsedTZ, err := time.LoadLocation(*input.Timezone) + if err != nil { + return nil, errors.New("Unknown timezone: " + *input.Timezone) + } + + tz = parsedTZ + } + + results, err := logs.Import(input.Importer, date, tz, input.ChannelName, input.Data) + if err != nil { + return nil, err + } + + newLogs := make([]models.Log, 0, len(results)) + for _, result := range results { + go func() { + changes.Submit("Log", "add", token.UserID, true, changekeys.Many(result.Log), result.Log) + changes.Submit("Post", "add", token.UserID, true, changekeys.Many(result.Log, result.Posts), result.Posts) + }() + + log, err := logs.UpdateCharacters(result.Log) + if err != nil { + log = result.Log + } + + newLogs = append(newLogs, log) + } + + return newLogs, nil +} + func (r *mutationResolver) EditLog(ctx context.Context, input input.LogEditInput) (models.Log, error) { token := auth.TokenFromContext(ctx) if !token.Authenticated() || !token.Permitted("log.edit") { diff --git a/graph2/schema/root.gql b/graph2/schema/root.gql index 51297bd..250dc48 100644 --- a/graph2/schema/root.gql +++ b/graph2/schema/root.gql @@ -95,6 +95,9 @@ type Mutation { # Add a new log addLog(input: LogAddInput!): Log! + # Import a log + importLog(input: LogImportInput!): [Log!]! + # Edit a log editLog(input: LogEditInput!): Log! diff --git a/graph2/schema/types/Log.gql b/graph2/schema/types/Log.gql index bb0c566..7c1ad57 100644 --- a/graph2/schema/types/Log.gql +++ b/graph2/schema/types/Log.gql @@ -100,4 +100,36 @@ input LogEditInput { input LogRemoveInput { # The id of the log to remove id: String! +} + +""" +Input for importLog mutation. +""" +input LogImportInput { + "The channel of the log, alwyas required." + channelName: String! + + "Which importer to use." + importer: LogImporter! + + "The timezone of the import, all log and post dates will be treated as it. It will be UTC if none is specified." + timezone: String + + "The date of the log, if not provided in the log body." + date: Date + + "The log body itself." + data: String! +} + +enum LogImporter { + """ + mIRC-like: This importer parses logs that copied from mIRC posts without spaces. + """ + MircLike + + """ + Forum log: This importer parses the format on the forum. The displayed log, not the post source. + """ + ForumLog } \ No newline at end of file diff --git a/internal/counter/counter.go b/internal/counter/counter.go index 3e90823..0ba3d09 100644 --- a/internal/counter/counter.go +++ b/internal/counter/counter.go @@ -30,6 +30,24 @@ func Next(category, name string) (int, error) { return doc.Value, nil } +// NextMany gets the next value of a counter, or an error if it hasn't, and increments by a specified value. +// Any value `returned` to `returned+(increment-1)` should then be safe to use. +func NextMany(category, name string, increment int) (int, error) { + id := category + "." + name + doc := counter{} + + _, err := collection.Find(bson.M{"_id": id}).Apply(mgo.Change{ + Update: bson.M{"$inc": bson.M{"value": 1}}, + Upsert: true, + ReturnNew: true, + }, &doc) + if err != nil { + return -1, err + } + + return doc.Value, nil +} + func init() { store.HandleInit(func(db *mgo.Database) { collection = db.C("core.counters") diff --git a/internal/importers/forumlog/logs.go b/internal/importers/forumlog/logs.go index 7b6ae95..8f11e9e 100644 --- a/internal/importers/forumlog/logs.go +++ b/internal/importers/forumlog/logs.go @@ -18,14 +18,14 @@ type ParsedLog struct { } // ParseLogs parses the logs from the data. -func ParseLogs(data string) ([]ParsedLog, error) { +func ParseLogs(data string, tz *time.Location) ([]ParsedLog, error) { metadata := ParseMetadata(data) results := make([]ParsedLog, 0, len(metadata["Date"])) scanner := bufio.NewScanner(strings.NewReader(data)) for i, dateStr := range metadata["Date"] { // Parse date - date, err := time.Parse("January 2, 2006", dateStr) + date, err := time.ParseInLocation("January 2, 2006", dateStr, tz) if err != nil { return nil, fmt.Errorf("Failed to parse date #%d: %#+v is not the in the correct format of \"January 2, 2006\"", i+1, dateStr) } diff --git a/internal/importers/forumlog/logs_test.go b/internal/importers/forumlog/logs_test.go index 168fa21..bf19008 100644 --- a/internal/importers/forumlog/logs_test.go +++ b/internal/importers/forumlog/logs_test.go @@ -12,7 +12,7 @@ import ( ) func TestParseLogs(t *testing.T) { - results, err := forumlog.ParseLogs(testLog) + results, err := forumlog.ParseLogs(testLog, time.UTC) if err != nil { t.Fatalf("Parse: %s", err) } @@ -23,9 +23,9 @@ func TestParseLogs(t *testing.T) { } func TestParseLogsErrors(t *testing.T) { - _, err1 := forumlog.ParseLogs(brokenLogNoPosts) - _, err2 := forumlog.ParseLogs(brokenLogBrokenPost) - _, err3 := forumlog.ParseLogs(brokenLogBrokenDate) + _, err1 := forumlog.ParseLogs(brokenLogNoPosts, time.UTC) + _, err2 := forumlog.ParseLogs(brokenLogBrokenPost, time.UTC) + _, err3 := forumlog.ParseLogs(brokenLogBrokenDate, time.UTC) t.Log("Should be about no posts:", err1) t.Log("Should be about a broken post:", err2) diff --git a/models/log-importer.go b/models/log-importer.go new file mode 100644 index 0000000..199407d --- /dev/null +++ b/models/log-importer.go @@ -0,0 +1,49 @@ +package models + +import ( + "fmt" + "io" + "strconv" +) + +// LogImporter describes a model related log importing. +type LogImporter string + +const ( + // LogImporterMircLike is a value of LogImporter + LogImporterMircLike LogImporter = "MircLike" + // LogImporterForumLog is a value of LogImporter + LogImporterForumLog LogImporter = "ForumLog" +) + +// IsValid returns true if the underlying string is one of the correct values. +func (e LogImporter) IsValid() bool { + switch e { + case LogImporterForumLog, LogImporterMircLike: + return true + } + return false +} + +func (e LogImporter) String() string { + return string(e) +} + +// UnmarshalGQL unmarshals the underlying graphql value. +func (e *LogImporter) UnmarshalGQL(v interface{}) error { + str, ok := v.(string) + if !ok { + return fmt.Errorf("enums must be strings") + } + + *e = LogImporter(str) + if !e.IsValid() { + return fmt.Errorf("%s is not a valid LogImporter", str) + } + return nil +} + +// MarshalGQL marshals the underlying graphql value. +func (e LogImporter) MarshalGQL(w io.Writer) { + fmt.Fprint(w, strconv.Quote(e.String())) +} diff --git a/models/logs/import.go b/models/logs/import.go new file mode 100644 index 0000000..f0b07be --- /dev/null +++ b/models/logs/import.go @@ -0,0 +1,93 @@ +package logs + +import ( + "errors" + "time" + + "git.aiterp.net/rpdata/api/internal/importers/mirclike" + + "git.aiterp.net/rpdata/api/models/posts" + + "git.aiterp.net/rpdata/api/models/channels" + + "git.aiterp.net/rpdata/api/internal/importers/forumlog" + + "git.aiterp.net/rpdata/api/models" +) + +// An ImportedLog contains data about an imported log. +type ImportedLog struct { + Log models.Log + Posts []models.Post +} + +// Import makes a log and posts object from different formats. +func Import(importer models.LogImporter, date time.Time, tz *time.Location, channelName string, data string) ([]ImportedLog, error) { + results := make([]ImportedLog, 0, 8) + + eventName := "" + if channel, err := channels.FindName(channelName); err != nil { + eventName = channel.EventName + } + + date = date.In(tz) + + switch importer { + case models.LogImporterMircLike: + { + if date.IsZero() { + return nil, errors.New("Date is not optional for mirc-like logs") + } + + parsedLog, parsedPosts, err := mirclike.ParseLog(data, date, true) + if err != nil { + return nil, err + } + + log, err := Add(parsedLog.Date, channelName, "", eventName, "", false) + if err != nil { + return nil, err + } + + posts, err := posts.AddMany(log, parsedPosts) + if err != nil { + return nil, err + } + + results = append(results, ImportedLog{ + Log: log, + Posts: posts, + }) + } + case models.LogImporterForumLog: + { + parseResults, err := forumlog.ParseLogs(data, tz) + if err != nil { + return nil, err + } + + for _, result := range parseResults { + log, err := Add(result.Log.Date, channelName, "", eventName, "", false) + if err != nil { + return nil, err + } + + posts, err := posts.AddMany(log, result.Posts) + if err != nil { + return nil, err + } + + results = append(results, ImportedLog{ + Log: log, + Posts: posts, + }) + } + } + default: + { + return nil, errors.New("Invalid importer: " + importer.String()) + } + } + + return results, nil +} diff --git a/models/posts/add-many.go b/models/posts/add-many.go new file mode 100644 index 0000000..ceb8f3f --- /dev/null +++ b/models/posts/add-many.go @@ -0,0 +1,34 @@ +package posts + +import ( + "git.aiterp.net/rpdata/api/internal/counter" + "git.aiterp.net/rpdata/api/models" +) + +// AddMany adds multiple posts in on query. Each post gets a new ID and is associated with the log. +func AddMany(log models.Log, posts []models.Post) ([]models.Post, error) { + docs := make([]interface{}, len(posts)) + + mutex.RLock() + defer mutex.RUnlock() + + startPosition, err := counter.NextMany("next_post_id", log.ShortID, len(posts)) + if err != nil { + return nil, err + } + + for i, post := range posts { + post.ID = generateID(post.Time) + post.LogID = log.ShortID + post.Position = startPosition + i + + docs[i] = post + } + + err = collection.Insert(docs...) + if err != nil { + return nil, err + } + + return posts, nil +} diff --git a/test.prof b/test.prof deleted file mode 100644 index 72784d4..0000000 Binary files a/test.prof and /dev/null differ