diff --git a/Gopkg.lock b/Gopkg.lock index ef1231b..1fe70c0 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -31,6 +31,12 @@ revision = "ace140f73450505f33e8b8418216792275ae82a7" version = "v1.35.0" +[[projects]] + name = "github.com/go-sql-driver/mysql" + packages = ["."] + revision = "d523deb1b23d913de5bdada721a6071e71283618" + version = "v1.4.0" + [[projects]] name = "github.com/graph-gophers/dataloader" packages = ["."] @@ -58,6 +64,15 @@ ] revision = "9ebf33af539ab8cb832c7107bc0a978ca8dbc0de" +[[projects]] + branch = "master" + name = "github.com/jmoiron/sqlx" + packages = [ + ".", + "reflectx" + ] + revision = "0dae4fefe7c0e190f7b5a78dac28a1c82cc8d849" + [[projects]] name = "github.com/minio/minio-go" packages = [ @@ -87,6 +102,12 @@ revision = "1949ddbfd147afd4d964a9f00b24eb291e0e7c38" version = "v1.0.2" +[[projects]] + name = "github.com/sadbox/mediawiki" + packages = ["."] + revision = "39fea8a1336076a961a300d1d95765dcd17e8a3c" + version = "v0.1" + [[projects]] name = "github.com/sirupsen/logrus" packages = ["."] @@ -143,9 +164,15 @@ revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" version = "v0.3.0" +[[projects]] + name = "google.golang.org/appengine" + packages = ["cloudsql"] + revision = "b1f26356af11148e710935ed1ac8a7f5702c7612" + version = "v1.1.0" + [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "0e9f91dc1e710ccd543842b55af8f6e4edbcb528246bb6d1e1e0c10d66328220" + inputs-digest = "f94e530148f893118f99f91ee446839f58784809949aac7e4a9c831b28b4f1c7" solver-name = "gps-cdcl" solver-version = 1 diff --git a/cmd/rpdata-as2storyimport/main.go b/cmd/rpdata-as2storyimport/main.go new file mode 100644 index 0000000..7e47f5e --- /dev/null +++ b/cmd/rpdata-as2storyimport/main.go @@ -0,0 +1,125 @@ +package main + +import ( + "flag" + "fmt" + "log" + "strings" + "time" + + "git.aiterp.net/rpdata/api/internal/store" + "git.aiterp.net/rpdata/api/model/story" + _ "github.com/go-sql-driver/mysql" + "github.com/jmoiron/sqlx" +) + +var flagHost = flag.String("host", "127.0.0.1:3306", "SQL host") +var flagDB = flag.String("database", "aitestory", "SQL database") +var flagUser = flag.String("user", "aitestory", "SQL user") +var flagPassword = flag.String("password", "", "SQL password") + +func main() { + flag.Parse() + + db, err := sqlx.Connect("mysql", fmt.Sprintf("%s:%s@(%s)/%s", *flagUser, *flagPassword, *flagHost, *flagDB)) + if err != nil { + log.Fatalln(err) + } + + err = db.Ping() + if err != nil { + log.Fatalln(err) + } + + store.Init() + + results := make([]storyResult, 0, 64) + rows, err := db.Queryx("SELECT * FROM page WHERE unlisted=0;") + for rows.Next() { + result := storyResult{} + err := rows.StructScan(&result) + if err != nil { + log.Fatalln(err) + } + results = append(results, result) + } + + tagResults := make([]tagResult, 0, 256) + rows, err = db.Queryx("SELECT page_id,type,name FROM page_tag LEFT JOIN tag ON tag_id=tag.id;") + for rows.Next() { + result := tagResult{} + err := rows.StructScan(&result) + if err != nil { + log.Fatalln(err) + } + tagResults = append(tagResults, result) + } + + for _, result := range results { + fictionalDate, err := time.Parse("2006-01-02 15:04:05", result.FictionalDate) + if err != nil { + if result.FictionalDate != "0000-00-00 00:00:00" { + log.Fatalln(err) + } + } + if fictionalDate.Year() < 1800 { + fictionalDate = time.Time{} + } + + publishDate, err := time.Parse("2006-01-02 15:04:05", result.PublishDate) + if err != nil { + log.Fatalln(err) + } + + tags := make([]story.Tag, 0, 8) + for _, tagResult := range tagResults { + if tagResult.PageID == result.ID { + tags = append(tags, story.Tag{Kind: tagResult.Type, Name: tagResult.Name}) + } + } + + story, err := story.New(result.Name, result.Author, result.Category, false, false, tags, publishDate, fictionalDate) + if err != nil { + log.Fatalln(err) + } + + title := result.Name + if strings.HasPrefix(result.Source, "#") { + firstNewline := strings.Index(result.Source, "\n") + title = result.Source[1:firstNewline] + result.Source = result.Source[firstNewline+1:] + } + + chapter, err := story.AddChapter(title, result.Author, result.Source, publishDate, fictionalDate) + if err != nil { + log.Fatalln(err) + } + + fmt.Println(result.ID, "->", story.ID, chapter.ID) + } +} + +type tagResult struct { + PageID string `db:"page_id"` + Type string `db:"type"` + Name string `db:"name"` +} + +type storyResult struct { + ID string `db:"id"` + Name string `db:"name"` + Author string `db:"author"` + Category string `db:"category"` + FictionalDate string `db:"fictional_date"` + PublishDate string `db:"publish_date"` + EditDate string `db:"edit_date"` + Unlisted bool `db:"unlisted"` + Dated bool `db:"dated"` + Spesific bool `db:"specific"` + Indexed bool `db:"indexed"` + Published bool `db:"published"` + Type string `db:"type"` + Source string `db:"source"` + Cache string `db:"cache"` + BackgroundURL *string `db:"background_url"` +} diff --git a/makefile b/makefile index 425e61e..a265253 100644 --- a/makefile +++ b/makefile @@ -1,17 +1,33 @@ INSTALL_PATH ?= ./build -build: +# Clean up previous builds, vendor directory and generated files +clean: + rm -rf ./vendor ./schema/bindata.go $(INSTALL_PATH) + +# Prepare the dev environment +setup: dep ensure go generate ./... go test ./... + +# Build the server (enough for a container/minimal install) +build-server: setup mkdir -p $(INSTALL_PATH)/usr/bin mkdir -p $(INSTALL_PATH)/etc/aiterp cp ./config.example.json $(INSTALL_PATH)/etc/aiterp/rpdata.json go build -ldflags="-s -w" -o $(INSTALL_PATH)/usr/bin/rpdata-graphiql ./cmd/rpdata-graphiql + +# Build the tools needed to port data over +build-tools: build-server go build -ldflags="-s -w" -o $(INSTALL_PATH)/usr/bin/rpdata-lb2charimport ./cmd/rpdata-lb2charimport go build -ldflags="-s -w" -o $(INSTALL_PATH)/usr/bin/rpdata-lb2logimport ./cmd/rpdata-lb2logimport go build -ldflags="-s -w" -o $(INSTALL_PATH)/usr/bin/rpdata-wikifileimport ./cmd/rpdata-wikifileimport go build -ldflags="-s -w" -o $(INSTALL_PATH)/usr/bin/rpdata-ensurechannels ./cmd/rpdata-ensurechannels + go build -ldflags="-s -w" -o $(INSTALL_PATH)/usr/bin/rpdata-as2storyimport ./cmd/rpdata-ensurechannels + +# Build all the things +build: build-server build-tools +# Install locally (requires access to /usr/bin, hence no dependency on build) install: cp $(INSTALL_PATH)/usr/bin/* /usr/local/bin/ \ No newline at end of file diff --git a/model/story/chapter.go b/model/story/chapter.go new file mode 100644 index 0000000..543b3ff --- /dev/null +++ b/model/story/chapter.go @@ -0,0 +1,92 @@ +package story + +import ( + "crypto/rand" + "encoding/binary" + "strconv" + "time" + + "git.aiterp.net/rpdata/api/internal/store" + "github.com/globalsign/mgo" + "github.com/globalsign/mgo/bson" +) + +var chapterCollection *mgo.Collection + +// A Chapter is a part of a story. +type Chapter struct { + ID string `bson:"_id"` + StoryID string `bson:"storyId"` + Title string `bson:"title"` + Author string `bson:"author"` + Source string `bson:"source"` + CreatedDate time.Time `bson:"createdDate"` + FictionalDate time.Time `bson:"fictionalDate,omitempty"` + EditedDate time.Time `bson:"editedDate"` +} + +// Edit edits a chapter, and updates EditedDate. While many Edit functions cheat if there's nothing to +// change, this functill will due to EditedDate. +func (chapter *Chapter) Edit(title, source *string, fictionalDate *time.Time) error { + now := time.Now() + changes := bson.M{"editedDate": now} + changed := *chapter + changed.EditedDate = now + + if title != nil && *title != chapter.Title { + changes["title"] = *title + changed.Title = *title + } + if source != nil && *source != chapter.Source { + changes["source"] = *source + changed.Source = *source + } + if fictionalDate != nil && *fictionalDate != chapter.FictionalDate { + changes["fictionalDate"] = *fictionalDate + changed.FictionalDate = *fictionalDate + } + + err := chapterCollection.UpdateId(chapter.ID, bson.M{"$set": changes}) + if err != nil { + return err + } + + *chapter = changed + + return nil +} + +// Remove removes a chapter. +func (chapter *Chapter) Remove() error { + return chapterCollection.RemoveId(chapter.ID) +} + +// makeChapterID makes a random chapter ID that's 24 characters long +func makeChapterID() string { + result := "SC" + offset := 0 + data := make([]byte, 32) + + rand.Read(data) + for len(result) < 24 { + result += strconv.FormatUint(binary.LittleEndian.Uint64(data[offset:]), 36) + offset += 8 + + if offset >= 32 { + rand.Read(data) + offset = 0 + } + } + + return result[:24] +} + +func init() { + store.HandleInit(func(db *mgo.Database) { + chapterCollection = db.C("story.chapters") + + chapterCollection.EnsureIndexKey("storyId") + chapterCollection.EnsureIndexKey("author") + chapterCollection.EnsureIndexKey("createdDate") + }) +} diff --git a/model/story/story.go b/model/story/story.go new file mode 100644 index 0000000..4cffa71 --- /dev/null +++ b/model/story/story.go @@ -0,0 +1,212 @@ +package story + +import ( + "crypto/rand" + "encoding/binary" + "errors" + "fmt" + "os" + "strconv" + "time" + + "git.aiterp.net/rpdata/api/internal/store" + "github.com/globalsign/mgo" + "github.com/globalsign/mgo/bson" +) + +var storyCollection *mgo.Collection + +// ErrTagAlreadyExists is an error returned by Story.AddTag +var ErrTagAlreadyExists = errors.New("Tag already exists") + +// ErrTagNotExists is an error returned by Story.RemoveTag +var ErrTagNotExists = errors.New("Tag does not exist") + +// A Story is user content that does not have a wiki-suitable format. Documents, new stories, short stories, and so on. +// The story model is a container for multiple chapters this time, in contrast to the previous version. +type Story struct { + ID string `bson:"_id"` + Author string `bson:"author"` + Name string `bson:"name"` + Category string `bson:"category"` + Open bool `bson:"open"` + Listed bool `bson:"listed"` + Tags []Tag `bson:"tags"` + CreatedDate time.Time `bson:"createdDate"` + FictionalDate time.Time `bson:"fictionalDate,omitempty"` + UpdatedDate time.Time `bson:"updatedDate"` +} + +// AddTag adds a tag to the story. It returns ErrTagAlreadyExists if the tag is already there +func (story *Story) AddTag(tag Tag) error { + for i := range story.Tags { + if story.Tags[i].Equal(tag) { + return ErrTagAlreadyExists + } + } + + err := storyCollection.UpdateId(story.ID, bson.M{"$push": bson.M{"tags": tag}}) + if err != nil { + return err + } + + story.Tags = append(story.Tags, tag) + + return nil +} + +// RemoveTag removes a tag to the story. It returns ErrTagNotExists if the tag does not exist. +func (story *Story) RemoveTag(tag Tag) error { + index := -1 + for i := range story.Tags { + if story.Tags[i].Equal(tag) { + index = i + break + } + } + if index == -1 { + return ErrTagNotExists + } + + err := storyCollection.UpdateId(story.ID, bson.M{"$pull": bson.M{"tags": tag}}) + if err != nil { + return err + } + + story.Tags = append(story.Tags[:index], story.Tags[index+1:]...) + + return nil +} + +// Edit edits the story, reflecting the new values in the story's struct values. If nothing will be +// changed, it will silently return without a database roundtrip. +func (story *Story) Edit(name, category *string, listed, open *bool, fictionalDate *time.Time) error { + changes := bson.M{} + changed := *story + + if name != nil && *name == story.Name { + changes["name"] = *name + changed.Name = *name + } + if category != nil && *category == story.Category { + changes["category"] = *category + changed.Name = *category + } + if listed != nil && *listed == story.Listed { + changes["listed"] = *listed + changed.Listed = *listed + } + if open != nil && *open == story.Open { + changes["open"] = *open + changed.Open = *open + } + if fictionalDate != nil && *fictionalDate == story.FictionalDate { + changes["fictionalDate"] = *fictionalDate + changed.FictionalDate = *fictionalDate + } + + if len(changes) == 0 { + return nil + } + + err := storyCollection.UpdateId(story.ID, bson.M{"$set": changes}) + if err != nil { + return err + } + + *story = changed + + return nil +} + +// Remove the story from the database +func (story *Story) Remove() error { + return storyCollection.RemoveId(story.ID) +} + +// AddChapter adds a chapter to the story. This does not enforce the `Open` setting, but it will log a warning if it +// occurs +func (story *Story) AddChapter(title, author, source string, createdDate, finctionalDate time.Time) (Chapter, error) { + if !story.Open && author != story.Author { + fmt.Fprintf(os.Stderr, "WARNING: AddChapter is breaking Open rules (story.id=%#+v, story.name=%#+v, chapter.author=%#+v, chapter.title=%#+v)", story.ID, story.Name, author, title) + } + + chapter := Chapter{ + ID: makeChapterID(), + StoryID: story.ID, + Title: title, + Author: author, + Source: source, + CreatedDate: createdDate, + FictionalDate: finctionalDate, + EditedDate: createdDate, + } + + err := chapterCollection.Insert(chapter) + if err != nil { + return Chapter{}, err + } + + if createdDate.After(story.UpdatedDate) { + if err := storyCollection.UpdateId(story.ID, bson.M{"$set": bson.M{"updatedDate": createdDate}}); err == nil { + story.UpdatedDate = createdDate + } + } + + return chapter, nil +} + +// New creates a new story. +func New(name, author, category string, listed, open bool, tags []Tag, createdDate, fictionalDate time.Time) (Story, error) { + story := Story{ + ID: makeStoryID(), + Name: name, + Author: author, + Category: category, + Listed: listed, + Open: open, + Tags: tags, + CreatedDate: createdDate, + FictionalDate: fictionalDate, + UpdatedDate: createdDate, + } + + err := storyCollection.Insert(story) + if err != nil { + return Story{}, err + } + + return story, nil +} + +// makeStoryID makes a random story ID that's 16 characters long +func makeStoryID() string { + result := "S" + offset := 0 + data := make([]byte, 32) + + rand.Read(data) + for len(result) < 16 { + result += strconv.FormatUint(binary.LittleEndian.Uint64(data[offset:]), 36) + offset += 8 + + if offset >= 32 { + rand.Read(data) + offset = 0 + } + } + + return result[:16] +} + +func init() { + store.HandleInit(func(db *mgo.Database) { + storyCollection = db.C("story.stories") + + storyCollection.EnsureIndexKey("tags") + storyCollection.EnsureIndexKey("author") + storyCollection.EnsureIndexKey("updatedDate") + storyCollection.EnsureIndexKey("fictionalDate") + storyCollection.EnsureIndexKey("listed") + }) +} diff --git a/model/story/tag.go b/model/story/tag.go new file mode 100644 index 0000000..6863aa3 --- /dev/null +++ b/model/story/tag.go @@ -0,0 +1,12 @@ +package story + +// A Tag associates a story with other content, like other stories, logs and more. +type Tag struct { + Kind string `bson:"kind"` + Name string `bson:"name"` +} + +// Equal returns true if the tags match one another. +func (tag *Tag) Equal(other Tag) bool { + return tag.Kind == other.Kind && tag.Name == other.Name +}