Added Story model, added rpdata-as2storyimport command

Gopkg.lock

@@ -31,6 +31,12 @@
   revision = "ace140f73450505f33e8b8418216792275ae82a7"
   version = "v1.35.0"
 
+[[projects]]
+  name = "github.com/go-sql-driver/mysql"
+  packages = ["."]
+  revision = "d523deb1b23d913de5bdada721a6071e71283618"
+  version = "v1.4.0"
+
 [[projects]]
   name = "github.com/graph-gophers/dataloader"
   packages = ["."]
@@ -58,6 +64,15 @@
   ]
   revision = "9ebf33af539ab8cb832c7107bc0a978ca8dbc0de"
 
+[[projects]]
+  branch = "master"
+  name = "github.com/jmoiron/sqlx"
+  packages = [
+    ".",
+    "reflectx"
+  ]
+  revision = "0dae4fefe7c0e190f7b5a78dac28a1c82cc8d849"
+
 [[projects]]
   name = "github.com/minio/minio-go"
   packages = [
@@ -87,6 +102,12 @@
   revision = "1949ddbfd147afd4d964a9f00b24eb291e0e7c38"
   version = "v1.0.2"
 
+[[projects]]
+  name = "github.com/sadbox/mediawiki"
+  packages = ["."]
+  revision = "39fea8a1336076a961a300d1d95765dcd17e8a3c"
+  version = "v0.1"
+
 [[projects]]
   name = "github.com/sirupsen/logrus"
   packages = ["."]
@@ -143,9 +164,15 @@
   revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
   version = "v0.3.0"
 
+[[projects]]
+  name = "google.golang.org/appengine"
+  packages = ["cloudsql"]
+  revision = "b1f26356af11148e710935ed1ac8a7f5702c7612"
+  version = "v1.1.0"
+
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "0e9f91dc1e710ccd543842b55af8f6e4edbcb528246bb6d1e1e0c10d66328220"
+  inputs-digest = "f94e530148f893118f99f91ee446839f58784809949aac7e4a9c831b28b4f1c7"
   solver-name = "gps-cdcl"
   solver-version = 1

cmd/rpdata-as2storyimport/main.go

@@ -0,0 +1,125 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"strings"
+	"time"
+
+	"git.aiterp.net/rpdata/api/internal/store"
+	"git.aiterp.net/rpdata/api/model/story"
+	_ "github.com/go-sql-driver/mysql"
+	"github.com/jmoiron/sqlx"
+)
+
+var flagHost = flag.String("host", "127.0.0.1:3306", "SQL host")
+var flagDB = flag.String("database", "aitestory", "SQL database")
+var flagUser = flag.String("user", "aitestory", "SQL user")
+var flagPassword = flag.String("password", "", "SQL password")
+
+func main() {
+	flag.Parse()
+
+	db, err := sqlx.Connect("mysql", fmt.Sprintf("%s:%s@(%s)/%s", *flagUser, *flagPassword, *flagHost, *flagDB))
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	err = db.Ping()
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	store.Init()
+
+	results := make([]storyResult, 0, 64)
+	rows, err := db.Queryx("SELECT * FROM page WHERE unlisted=0;")
+	for rows.Next() {
+		result := storyResult{}
+		err := rows.StructScan(&result)
+		if err != nil {
+			log.Fatalln(err)
+		}
+		results = append(results, result)
+	}
+
+	tagResults := make([]tagResult, 0, 256)
+	rows, err = db.Queryx("SELECT page_id,type,name FROM page_tag LEFT JOIN tag ON tag_id=tag.id;")
+	for rows.Next() {
+		result := tagResult{}
+		err := rows.StructScan(&result)
+		if err != nil {
+			log.Fatalln(err)
+		}
+		tagResults = append(tagResults, result)
+	}
+
+	for _, result := range results {
+		fictionalDate, err := time.Parse("2006-01-02 15:04:05", result.FictionalDate)
+		if err != nil {
+			if result.FictionalDate != "0000-00-00 00:00:00" {
+				log.Fatalln(err)
+			}
+		}
+		if fictionalDate.Year() < 1800 {
+			fictionalDate = time.Time{}
+		}
+
+		publishDate, err := time.Parse("2006-01-02 15:04:05", result.PublishDate)
+		if err != nil {
+			log.Fatalln(err)
+		}
+
+		tags := make([]story.Tag, 0, 8)
+		for _, tagResult := range tagResults {
+			if tagResult.PageID == result.ID {
+				tags = append(tags, story.Tag{Kind: tagResult.Type, Name: tagResult.Name})
+			}
+		}
+
+		story, err := story.New(result.Name, result.Author, result.Category, false, false, tags, publishDate, fictionalDate)
+		if err != nil {
+			log.Fatalln(err)
+		}
+
+		title := result.Name
+		if strings.HasPrefix(result.Source, "#") {
+			firstNewline := strings.Index(result.Source, "\n")
+			title = result.Source[1:firstNewline]
+			result.Source = result.Source[firstNewline+1:]
+		}
+
+		chapter, err := story.AddChapter(title, result.Author, result.Source, publishDate, fictionalDate)
+		if err != nil {
+			log.Fatalln(err)
+		}
+
+		fmt.Println(result.ID, "->", story.ID, chapter.ID)
+	}
+}
+
+type tagResult struct {
+	PageID string `db:"page_id"`
+	Type   string `db:"type"`
+	Name   string `db:"name"`
+}
+
+type storyResult struct {
+	ID            string  `db:"id"`
+	Name          string  `db:"name"`
+	Author        string  `db:"author"`
+	Category      string  `db:"category"`
+	FictionalDate string  `db:"fictional_date"`
+	PublishDate   string  `db:"publish_date"`
+	EditDate      string  `db:"edit_date"`
+	Unlisted      bool    `db:"unlisted"`
+	Dated         bool    `db:"dated"`
+	Spesific      bool    `db:"specific"`
+	Indexed       bool    `db:"indexed"`
+	Published     bool    `db:"published"`
+	Type          string  `db:"type"`
+	Source        string  `db:"source"`
+	Cache         string  `db:"cache"`
+	BackgroundURL *string `db:"background_url"`
+}

makefile

@@ -1,17 +1,33 @@
 INSTALL_PATH ?= ./build
 
-build:
+# Clean up previous builds, vendor directory and generated files
+clean:
+	rm -rf ./vendor ./schema/bindata.go $(INSTALL_PATH)
+
+# Prepare the dev environment
+setup:
 	dep ensure
 	go generate ./...
 	go test ./...
+
+# Build the server (enough for a container/minimal install)
+build-server: setup
 	mkdir -p $(INSTALL_PATH)/usr/bin
 	mkdir -p $(INSTALL_PATH)/etc/aiterp
 	cp ./config.example.json $(INSTALL_PATH)/etc/aiterp/rpdata.json
 	go build -ldflags="-s -w" -o $(INSTALL_PATH)/usr/bin/rpdata-graphiql ./cmd/rpdata-graphiql
+
+# Build the tools needed to port data over
+build-tools: build-server
 	go build -ldflags="-s -w" -o $(INSTALL_PATH)/usr/bin/rpdata-lb2charimport ./cmd/rpdata-lb2charimport
 	go build -ldflags="-s -w" -o $(INSTALL_PATH)/usr/bin/rpdata-lb2logimport ./cmd/rpdata-lb2logimport
 	go build -ldflags="-s -w" -o $(INSTALL_PATH)/usr/bin/rpdata-wikifileimport ./cmd/rpdata-wikifileimport
 	go build -ldflags="-s -w" -o $(INSTALL_PATH)/usr/bin/rpdata-ensurechannels ./cmd/rpdata-ensurechannels
+	go build -ldflags="-s -w" -o $(INSTALL_PATH)/usr/bin/rpdata-as2storyimport ./cmd/rpdata-ensurechannels
+
+# Build all the things
+build: build-server build-tools
 
+# Install locally (requires access to /usr/bin, hence no dependency on build)
 install:
 	cp $(INSTALL_PATH)/usr/bin/* /usr/local/bin/

model/story/chapter.go

@@ -0,0 +1,92 @@
+package story
+
+import (
+	"crypto/rand"
+	"encoding/binary"
+	"strconv"
+	"time"
+
+	"git.aiterp.net/rpdata/api/internal/store"
+	"github.com/globalsign/mgo"
+	"github.com/globalsign/mgo/bson"
+)
+
+var chapterCollection *mgo.Collection
+
+// A Chapter is a part of a story.
+type Chapter struct {
+	ID            string    `bson:"_id"`
+	StoryID       string    `bson:"storyId"`
+	Title         string    `bson:"title"`
+	Author        string    `bson:"author"`
+	Source        string    `bson:"source"`
+	CreatedDate   time.Time `bson:"createdDate"`
+	FictionalDate time.Time `bson:"fictionalDate,omitempty"`
+	EditedDate    time.Time `bson:"editedDate"`
+}
+
+// Edit edits a chapter, and updates EditedDate. While many Edit functions cheat if there's nothing to
+// change, this functill will due to EditedDate.
+func (chapter *Chapter) Edit(title, source *string, fictionalDate *time.Time) error {
+	now := time.Now()
+	changes := bson.M{"editedDate": now}
+	changed := *chapter
+	changed.EditedDate = now
+
+	if title != nil && *title != chapter.Title {
+		changes["title"] = *title
+		changed.Title = *title
+	}
+	if source != nil && *source != chapter.Source {
+		changes["source"] = *source
+		changed.Source = *source
+	}
+	if fictionalDate != nil && *fictionalDate != chapter.FictionalDate {
+		changes["fictionalDate"] = *fictionalDate
+		changed.FictionalDate = *fictionalDate
+	}
+
+	err := chapterCollection.UpdateId(chapter.ID, bson.M{"$set": changes})
+	if err != nil {
+		return err
+	}
+
+	*chapter = changed
+
+	return nil
+}
+
+// Remove removes a chapter.
+func (chapter *Chapter) Remove() error {
+	return chapterCollection.RemoveId(chapter.ID)
+}
+
+// makeChapterID makes a random chapter ID that's 24 characters long
+func makeChapterID() string {
+	result := "SC"
+	offset := 0
+	data := make([]byte, 32)
+
+	rand.Read(data)
+	for len(result) < 24 {
+		result += strconv.FormatUint(binary.LittleEndian.Uint64(data[offset:]), 36)
+		offset += 8
+
+		if offset >= 32 {
+			rand.Read(data)
+			offset = 0
+		}
+	}
+
+	return result[:24]
+}
+
+func init() {
+	store.HandleInit(func(db *mgo.Database) {
+		chapterCollection = db.C("story.chapters")
+
+		chapterCollection.EnsureIndexKey("storyId")
+		chapterCollection.EnsureIndexKey("author")
+		chapterCollection.EnsureIndexKey("createdDate")
+	})
+}

model/story/story.go

@@ -0,0 +1,212 @@
+package story
+
+import (
+	"crypto/rand"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"os"
+	"strconv"
+	"time"
+
+	"git.aiterp.net/rpdata/api/internal/store"
+	"github.com/globalsign/mgo"
+	"github.com/globalsign/mgo/bson"
+)
+
+var storyCollection *mgo.Collection
+
+// ErrTagAlreadyExists is an error returned by Story.AddTag
+var ErrTagAlreadyExists = errors.New("Tag already exists")
+
+// ErrTagNotExists is an error returned by Story.RemoveTag
+var ErrTagNotExists = errors.New("Tag does not exist")
+
+// A Story is user content that does not have a wiki-suitable format. Documents, new stories, short stories, and so on.
+// The story model is a container for multiple chapters this time, in contrast to the previous version.
+type Story struct {
+	ID            string    `bson:"_id"`
+	Author        string    `bson:"author"`
+	Name          string    `bson:"name"`
+	Category      string    `bson:"category"`
+	Open          bool      `bson:"open"`
+	Listed        bool      `bson:"listed"`
+	Tags          []Tag     `bson:"tags"`
+	CreatedDate   time.Time `bson:"createdDate"`
+	FictionalDate time.Time `bson:"fictionalDate,omitempty"`
+	UpdatedDate   time.Time `bson:"updatedDate"`
+}
+
+// AddTag adds a tag to the story. It returns ErrTagAlreadyExists if the tag is already there
+func (story *Story) AddTag(tag Tag) error {
+	for i := range story.Tags {
+		if story.Tags[i].Equal(tag) {
+			return ErrTagAlreadyExists
+		}
+	}
+
+	err := storyCollection.UpdateId(story.ID, bson.M{"$push": bson.M{"tags": tag}})
+	if err != nil {
+		return err
+	}
+
+	story.Tags = append(story.Tags, tag)
+
+	return nil
+}
+
+// RemoveTag removes a tag to the story. It returns ErrTagNotExists if the tag does not exist.
+func (story *Story) RemoveTag(tag Tag) error {
+	index := -1
+	for i := range story.Tags {
+		if story.Tags[i].Equal(tag) {
+			index = i
+			break
+		}
+	}
+	if index == -1 {
+		return ErrTagNotExists
+	}
+
+	err := storyCollection.UpdateId(story.ID, bson.M{"$pull": bson.M{"tags": tag}})
+	if err != nil {
+		return err
+	}
+
+	story.Tags = append(story.Tags[:index], story.Tags[index+1:]...)
+
+	return nil
+}
+
+// Edit edits the story, reflecting the new values in the story's struct values. If nothing will be
+// changed, it will silently return without a database roundtrip.
+func (story *Story) Edit(name, category *string, listed, open *bool, fictionalDate *time.Time) error {
+	changes := bson.M{}
+	changed := *story
+
+	if name != nil && *name == story.Name {
+		changes["name"] = *name
+		changed.Name = *name
+	}
+	if category != nil && *category == story.Category {
+		changes["category"] = *category
+		changed.Name = *category
+	}
+	if listed != nil && *listed == story.Listed {
+		changes["listed"] = *listed
+		changed.Listed = *listed
+	}
+	if open != nil && *open == story.Open {
+		changes["open"] = *open
+		changed.Open = *open
+	}
+	if fictionalDate != nil && *fictionalDate == story.FictionalDate {
+		changes["fictionalDate"] = *fictionalDate
+		changed.FictionalDate = *fictionalDate
+	}
+
+	if len(changes) == 0 {
+		return nil
+	}
+
+	err := storyCollection.UpdateId(story.ID, bson.M{"$set": changes})
+	if err != nil {
+		return err
+	}
+
+	*story = changed
+
+	return nil
+}
+
+// Remove the story from the database
+func (story *Story) Remove() error {
+	return storyCollection.RemoveId(story.ID)
+}
+
+// AddChapter adds a chapter to the story. This does not enforce the `Open` setting, but it will log a warning if it
+// occurs
+func (story *Story) AddChapter(title, author, source string, createdDate, finctionalDate time.Time) (Chapter, error) {
+	if !story.Open && author != story.Author {
+		fmt.Fprintf(os.Stderr, "WARNING: AddChapter is breaking Open rules (story.id=%#+v, story.name=%#+v, chapter.author=%#+v, chapter.title=%#+v)", story.ID, story.Name, author, title)
+	}
+
+	chapter := Chapter{
+		ID:            makeChapterID(),
+		StoryID:       story.ID,
+		Title:         title,
+		Author:        author,
+		Source:        source,
+		CreatedDate:   createdDate,
+		FictionalDate: finctionalDate,
+		EditedDate:    createdDate,
+	}
+
+	err := chapterCollection.Insert(chapter)
+	if err != nil {
+		return Chapter{}, err
+	}
+
+	if createdDate.After(story.UpdatedDate) {
+		if err := storyCollection.UpdateId(story.ID, bson.M{"$set": bson.M{"updatedDate": createdDate}}); err == nil {
+			story.UpdatedDate = createdDate
+		}
+	}
+
+	return chapter, nil
+}
+
+// New creates a new story.
+func New(name, author, category string, listed, open bool, tags []Tag, createdDate, fictionalDate time.Time) (Story, error) {
+	story := Story{
+		ID:            makeStoryID(),
+		Name:          name,
+		Author:        author,
+		Category:      category,
+		Listed:        listed,
+		Open:          open,
+		Tags:          tags,
+		CreatedDate:   createdDate,
+		FictionalDate: fictionalDate,
+		UpdatedDate:   createdDate,
+	}
+
+	err := storyCollection.Insert(story)
+	if err != nil {
+		return Story{}, err
+	}
+
+	return story, nil
+}
+
+// makeStoryID makes a random story ID that's 16 characters long
+func makeStoryID() string {
+	result := "S"
+	offset := 0
+	data := make([]byte, 32)
+
+	rand.Read(data)
+	for len(result) < 16 {
+		result += strconv.FormatUint(binary.LittleEndian.Uint64(data[offset:]), 36)
+		offset += 8
+
+		if offset >= 32 {
+			rand.Read(data)
+			offset = 0
+		}
+	}
+
+	return result[:16]
+}
+
+func init() {
+	store.HandleInit(func(db *mgo.Database) {
+		storyCollection = db.C("story.stories")
+
+		storyCollection.EnsureIndexKey("tags")
+		storyCollection.EnsureIndexKey("author")
+		storyCollection.EnsureIndexKey("updatedDate")
+		storyCollection.EnsureIndexKey("fictionalDate")
+		storyCollection.EnsureIndexKey("listed")
+	})
+}

model/story/tag.go

@@ -0,0 +1,12 @@
+package story
+
+// A Tag associates a story with other content, like other stories, logs and more.
+type Tag struct {
+	Kind string `bson:"kind"`
+	Name string `bson:"name"`
+}
+
+// Equal returns true if the tags match one another.
+func (tag *Tag) Equal(other Tag) bool {
+	return tag.Kind == other.Kind && tag.Name == other.Name
+}