More destruction. Utils too, since they cause weird go mod shenanigans.

Dockerfile

@@ -12,11 +12,6 @@ RUN go generate ./...
 # Build server and tools (CGO disabled to make them 100% static)
 ENV CGO_ENABLED 0
 RUN go build -installsuffix cgo -ldflags="-s -w" -o /binaries/rpdata-server ./cmd/rpdata-server
-RUN go build -installsuffix cgo -ldflags="-s -w" -o /binaries/rpdata-lb2charimport ./cmd/rpdata-lb2charimport
-RUN go build -installsuffix cgo -ldflags="-s -w" -o /binaries/rpdata-lb2logimport ./cmd/rpdata-lb2logimport
-# TODO: RUN go build -installsuffix cgo -ldflags="-s -w" -o /binaries/rpdata-wikifileimport ./cmd/rpdata-wikifileimport
-RUN go build -installsuffix cgo -ldflags="-s -w" -o /binaries/rpdata-ensurechannels ./cmd/rpdata-ensurechannels
-RUN go build -installsuffix cgo -ldflags="-s -w" -o /binaries/rpdata-as2storyimport ./cmd/rpdata-as2storyimport
 
 ## 2. Distribute
 # Use alpine linux

cmd/rpdata-as2storyimport/main.go

@@ -1,139 +0,0 @@
-package main
-
-import (
-	"flag"
-	"fmt"
-	"log"
-	"strings"
-	"time"
-
-	"git.aiterp.net/rpdata/api/internal/store"
-	"git.aiterp.net/rpdata/api/models"
-	"git.aiterp.net/rpdata/api/models/chapters"
-	"git.aiterp.net/rpdata/api/models/stories"
-	_ "github.com/go-sql-driver/mysql"
-	"github.com/jmoiron/sqlx"
-)
-
-var flagHost = flag.String("host", "127.0.0.1:3306", "SQL host")
-var flagDB = flag.String("database", "aitestory", "SQL database")
-var flagUser = flag.String("user", "aitestory", "SQL user")
-var flagPassword = flag.String("password", "", "SQL password")
-
-func main() {
-	flag.Parse()
-
-	db, err := sqlx.Connect("mysql", fmt.Sprintf("%s:%s@(%s)/%s", *flagUser, *flagPassword, *flagHost, *flagDB))
-	if err != nil {
-		log.Fatalln(err)
-	}
-
-	err = db.Ping()
-	if err != nil {
-		log.Fatalln(err)
-	}
-
-	store.Init()
-
-	results := make([]storyResult, 0, 64)
-	rows, err := db.Queryx("SELECT * FROM page WHERE unlisted=0;")
-	for rows.Next() {
-		result := storyResult{}
-		err := rows.StructScan(&result)
-		if err != nil {
-			log.Fatalln(err)
-		}
-		results = append(results, result)
-	}
-
-	tagResults := make([]tagResult, 0, 256)
-	rows, err = db.Queryx("SELECT page_id,type,name FROM page_tag LEFT JOIN tag ON tag_id=tag.id;")
-	for rows.Next() {
-		result := tagResult{}
-		err := rows.StructScan(&result)
-		if err != nil {
-			log.Fatalln(err)
-		}
-		tagResults = append(tagResults, result)
-	}
-
-	for _, result := range results {
-		fictionalDate, err := time.Parse("2006-01-02 15:04:05", result.FictionalDate)
-		if err != nil {
-			if result.FictionalDate != "0000-00-00 00:00:00" {
-				log.Fatalln(err)
-			}
-		}
-		if fictionalDate.Year() < 1800 {
-			fictionalDate = time.Time{}
-		}
-
-		publishDate, err := time.Parse("2006-01-02 15:04:05", result.PublishDate)
-		if err != nil {
-			log.Fatalln(err)
-		}
-
-		tags := make([]models.Tag, 0, 8)
-		for _, tagResult := range tagResults {
-			if tagResult.PageID == result.ID {
-				tags = append(tags, models.Tag{Kind: models.TagKind(tagResult.Type), Name: tagResult.Name})
-			}
-		}
-
-		category := models.StoryCategory(result.Category)
-		if !category.IsValid() {
-			log.Println(result.Name, "does not have a valid category:", result.Category)
-			continue
-		}
-
-		story, err := stories.Add(result.Name, result.Author[5:], category, true, false, tags, publishDate, fictionalDate)
-		if err != nil {
-			log.Fatalln(err)
-		}
-
-		// Change the story title
-		title := result.Name
-		if strings.HasPrefix(result.Source, "#") {
-			firstNewline := strings.Index(result.Source, "\n")
-			result.Name = strings.Replace(result.Source[1:firstNewline], "\r", "", -1)
-			result.Source = result.Source[firstNewline+1:]
-
-			result.Source = strings.Replace(result.Source, "\r\n", "\n", -1)
-			if strings.HasPrefix(result.Source, "\r") || strings.HasPrefix(result.Source, "\n") {
-				result.Source = result.Source[1:]
-			}
-		}
-
-		chapter, err := chapters.Add(story, title, result.Author[5:], result.Source, publishDate, &fictionalDate, models.ChapterCommentModeDisabled)
-		if err != nil {
-			log.Fatalln(err)
-		}
-
-		fmt.Println(result.ID, "->", story.ID, chapter.ID)
-	}
-}
-
-type tagResult struct {
-	PageID string `db:"page_id"`
-	Type   string `db:"type"`
-	Name   string `db:"name"`
-}
-
-type storyResult struct {
-	ID            string  `db:"id"`
-	Name          string  `db:"name"`
-	Author        string  `db:"author"`
-	Category      string  `db:"category"`
-	FictionalDate string  `db:"fictional_date"`
-	PublishDate   string  `db:"publish_date"`
-	EditDate      string  `db:"edit_date"`
-	Unlisted      bool    `db:"unlisted"`
-	Dated         bool    `db:"dated"`
-	Spesific      bool    `db:"specific"`
-	Indexed       bool    `db:"indexed"`
-	Published     bool    `db:"published"`
-	Type          string  `db:"type"`
-	Source        string  `db:"source"`
-	Cache         string  `db:"cache"`
-	BackgroundURL *string `db:"background_url"`
-}

cmd/rpdata-ensurechannels/main.go

@@ -1,41 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"os"
-
-	"git.aiterp.net/rpdata/api/internal/store"
-	"git.aiterp.net/rpdata/api/models/channels"
-	"git.aiterp.net/rpdata/api/models/logs"
-)
-
-func main() {
-	err := store.Init()
-	if err != nil {
-		fmt.Fprintln(os.Stderr, err)
-		return
-	}
-
-	logs, err := logs.List(nil)
-	if err != nil {
-		fmt.Fprintln(os.Stderr, err)
-		return
-	}
-
-	added := make(map[string]bool, 1024)
-	for _, log := range logs {
-		if added[log.ChannelName] {
-			continue
-		}
-
-		_, err := channels.Ensure(log.ChannelName, false)
-		if err != nil {
-			fmt.Fprintln(os.Stderr, log.ID, err)
-			continue
-		}
-
-		added[log.ChannelName] = true
-
-		fmt.Println(log.ChannelName, "ensured")
-	}
-}

cmd/rpdata-lb2charimport/.gitignore

@@ -1,3 +0,0 @@
-characters.json
-characters.cson
-debug

cmd/rpdata-lb2charimport/data.go

@@ -1,55 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"io"
-	"strings"
-)
-
-type charInfo struct {
-	Nicks     []string `json:"nicks"`
-	Name      string   `json:"name"`
-	Author    string   `json:"player"`
-	ShortName string   `json:"first"`
-}
-
-func load(reader io.Reader) ([]charInfo, error) {
-	data := make(map[string]interface{})
-	err := json.NewDecoder(reader).Decode(&data)
-	if err != nil {
-		return nil, err
-	}
-
-	links := make(map[string]string, len(data))
-	infos := make([]charInfo, 0, 64)
-
-	for key, value := range data {
-		if info, ok := value.(map[string]interface{}); ok {
-			name := info["name"].(string)
-			author := info["player"].(string)
-			shortName, ok := info["first"].(string)
-			if !ok {
-				shortName = strings.SplitN(name, " ", 2)[0]
-			}
-
-			infos = append(infos, charInfo{
-				Nicks:     []string{key},
-				Name:      name,
-				Author:    author,
-				ShortName: shortName,
-			})
-		} else if nick, ok := value.(string); ok {
-			links[key] = nick
-		}
-	}
-
-	for key, value := range links {
-		for i := range infos {
-			if infos[i].Nicks[0] == value {
-				infos[i].Nicks = append(infos[i].Nicks, key)
-			}
-		}
-	}
-
-	return infos, nil
-}

cmd/rpdata-lb2charimport/main.go

@@ -1,62 +0,0 @@
-package main
-
-import (
-	"flag"
-	"log"
-	"os"
-
-	"git.aiterp.net/rpdata/api/internal/store"
-	"git.aiterp.net/rpdata/api/models/characters"
-)
-
-var fileName = flag.String("file", "./characters.json", "json file to load")
-
-func main() {
-	file, err := os.Open(*fileName)
-	if err != nil {
-		log.Fatalln("Open:", err)
-	}
-
-	infos, err := load(file)
-	if err != nil {
-		log.Fatalln("Parse:", err)
-	}
-
-	err = store.Init()
-	if err != nil {
-		log.Fatalln("Store:", err)
-	}
-
-	charsAdded, charsFailed := 0, 0
-	nicksAdded, nicksFailed := 0, 0
-
-	for _, info := range infos {
-		char, err := characters.Add(info.Nicks[0], info.Name, info.ShortName, info.Author, "")
-		if err != nil {
-			log.Println(info.Nicks[0], "failed to insert:", err)
-			charsFailed++
-
-			char, err = characters.FindNick(info.Nicks[0])
-			if err != nil {
-				continue
-			}
-		} else {
-			log.Println(info.Nicks[0], "added")
-			charsAdded++
-		}
-
-		for _, alt := range info.Nicks[1:] {
-			char, err = characters.AddNick(char, alt)
-			if err != nil {
-				log.Println(info.Nicks[0], "failed to add nick", alt, "error:", err)
-				nicksFailed++
-			} else {
-				log.Println(info.Nicks[0], "addded nick", alt)
-				nicksAdded++
-			}
-		}
-	}
-
-	log.Printf("Characters – %d/%d", charsAdded, charsFailed+charsAdded)
-	log.Printf("Alt. Nicks – %d/%d", nicksAdded, nicksFailed+nicksAdded)
-}

cmd/rpdata-lb2logimport/.gitignore

@@ -1,2 +0,0 @@
-logs
-debug

cmd/rpdata-lb2logimport/line.go

@@ -1,85 +0,0 @@
-package main
-
-import (
-	"bufio"
-	"errors"
-	"io"
-	"strconv"
-	"strings"
-	"time"
-)
-
-type logLine struct {
-	Verb string
-	Args []string
-	Text string
-}
-
-func parseFile(reader io.Reader) []logLine {
-	bufReader := bufio.NewReader(reader)
-	results := make([]logLine, 0, 512)
-
-	for {
-		line, err := bufReader.ReadString('\n')
-		if err == io.EOF {
-			if len(line) < 2 {
-				break
-			}
-		} else if err != nil {
-			break
-		}
-
-		if len(line) <= 2 {
-			continue
-		}
-
-		line = strings.Replace(line, "\n", "", 1)
-		line = strings.Replace(line, "\r", "", 1)
-
-		results = append(results, parseLine(line))
-	}
-
-	return results
-}
-
-func parseLine(line string) logLine {
-	textSplit := strings.SplitN(line, " :", 2)
-	tokens := strings.Split(textSplit[0], " ")
-
-	ll := logLine{
-		Verb: tokens[0],
-	}
-
-	if len(tokens) > 1 {
-		ll.Args = tokens[1:]
-	}
-
-	if len(textSplit) > 1 {
-		ll.Text = textSplit[1]
-	}
-
-	return ll
-}
-
-func parseFilename(fname string) (date time.Time, channel string, err error) {
-	date, err = time.ParseInLocation("2006-01-02_150405", fname[:17], time.Local)
-	if err != nil {
-		return
-	}
-
-	ms, err := strconv.Atoi(fname[17:20])
-	if err != nil {
-		return
-	}
-
-	date = date.Add(time.Duration(ms) * time.Millisecond)
-
-	if len(fname) < 23 {
-		err = errors.New("filename too short")
-		return
-	}
-
-	channel = fname[21:]
-
-	return
-}

cmd/rpdata-lb2logimport/main.go

@@ -1,110 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"log"
-	"os"
-	"path"
-	"strings"
-	"time"
-
-	"git.aiterp.net/rpdata/api/models/posts"
-
-	"git.aiterp.net/rpdata/api/internal/store"
-	"git.aiterp.net/rpdata/api/models/logs"
-)
-
-var prefixReplacer = strings.NewReplacer("+", "", "@", "", "!", "", "%", "")
-
-func main() {
-	err := store.Init()
-	if err != nil {
-		log.Fatalln(err)
-	}
-
-	for _, filepath := range os.Args[1:] {
-		name := strings.Replace(path.Base(filepath), ".txt", "", 1)
-		file, err := os.Open(filepath)
-		if err != nil {
-			log.Println(filepath, err)
-			continue
-		}
-		logLines := parseFile(file)
-		file.Close()
-
-		// Get title and event
-		title := ""
-		event := ""
-		for _, line := range logLines {
-			if line.Verb == "TITLE" {
-				title = line.Text
-			}
-			if line.Verb == "TAG" {
-				event = line.Text
-			}
-		}
-
-		date, channel, err := parseFilename(name)
-		if err != nil {
-			log.Fatalln(err)
-		}
-
-		l, err := logs.Add(date, channel, title, event, "", false)
-		if err != nil {
-			log.Println(err)
-			continue
-		}
-
-		_, err = posts.Add(l, time.Now(), "annotation.info", "rpdata-lb2logimport", "This logfile is imported from aitelogs2 and may contain errors or wrong timestamps.")
-		if err != nil {
-			log.Println(err)
-		}
-
-		for _, line := range logLines {
-			if line.Verb != "CHARS" {
-				continue
-			}
-
-			_, err = posts.Add(l, l.Date, "chars", prefixReplacer.Replace(line.Args[0]), line.Text)
-			if err != nil {
-				log.Println(err)
-			}
-		}
-
-		for _, line := range logLines {
-			if line.Verb != "SCENE" && line.Verb != "ACTION" && line.Verb != "TEXT" {
-				continue
-			}
-
-			postTime, err := time.ParseInLocation("2006-01-02 15:04:05", date.Format("2006-01-02")+" "+line.Args[1], time.Local)
-			diff := postTime.Sub(date)
-			if err != nil {
-				log.Println(err)
-				continue
-			}
-			if diff < 0 {
-				if diff > -time.Second {
-					postTime = postTime.Add(diff)
-				} else {
-					postTime = postTime.Add(time.Hour * 24)
-				}
-			}
-
-			if line.Args[0][0] == '=' {
-				line.Verb = "SCENE"
-			}
-
-			_, err = posts.Add(l, postTime, strings.ToLower(line.Verb), prefixReplacer.Replace(line.Args[0]), line.Text)
-			if err != nil {
-				log.Println(err)
-			}
-		}
-
-		l, err = logs.UpdateCharacters(l, nil)
-		if err != nil {
-			log.Println(err)
-		}
-
-		fmt.Println(l.ID, "completed")
-	}
-}

cmd/rpdata-wikifileimport/main.go

@@ -1,77 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"log"
-	"net/http"
-	"time"
-
-	"git.aiterp.net/rpdata/api/internal/config"
-	"git.aiterp.net/rpdata/api/internal/store"
-	"git.aiterp.net/rpdata/api/models/files"
-)
-
-func main() {
-	client := http.Client{Timeout: time.Second * 30}
-
-	res, err := client.Get(config.Global().Wiki.URL + "?action=query&list=allimages&ailimit=5&aiprop=dimensions|mime|url|user|timestamp&format=json&ailimit=500")
-	if err != nil {
-		log.Fatalln(err)
-	}
-
-	if res.StatusCode != 200 {
-		log.Fatalln(res.Status)
-	}
-
-	data := ResponseData{}
-	err = json.NewDecoder(res.Body).Decode(&data)
-	if err != nil {
-		log.Fatalln(err)
-	}
-
-	store.Init()
-
-	for _, info := range data.Query.Allimages {
-		existing, err := files.FindName("wiki", info.Name, info.User)
-		if err == nil && existing.ID != "" {
-			log.Println("Skipping", info.Name, "because it already exists")
-			continue
-		}
-
-		ts, err := time.Parse(time.RFC3339, info.Timestamp)
-		if err != nil {
-			log.Println("Skipping", info.Name, "error:", err)
-			continue
-		}
-
-		file, err := files.Insert(info.Name, "wiki", info.Mime, info.User, ts, info.Size, info.URL)
-		if err != nil {
-			log.Println("Skipping", info.Name, "error:", err)
-			continue
-		}
-
-		fmt.Println(file.Name, "inserted ( id:", file.ID, ")")
-	}
-}
-
-// The ResponseData from the wiki looks like this
-type ResponseData struct {
-	Batchcomplete string `json:"batchcomplete"`
-	Query         struct {
-		Allimages []struct {
-			Descriptionshorturl string `json:"descriptionshorturl"`
-			Descriptionurl      string `json:"descriptionurl"`
-			Height              int    `json:"height"`
-			Mime                string `json:"mime"`
-			Name                string `json:"name"`
-			Ns                  int    `json:"ns"`
-			Size                int64  `json:"size"`
-			Timestamp           string `json:"timestamp"`
-			Title               string `json:"title"`
-			URL                 string `json:"url"`
-			User                string `json:"user"`
-			Width               int    `json:"width"`
-		} `json:"allimages"`
-	} `json:"query"`
-}

go.mod

@@ -8,15 +8,10 @@ require (
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/globalsign/mgo v0.0.0-20180403085842-f76e4f9da92e
 	github.com/go-ini/ini v1.35.0 // indirect
-	github.com/go-sql-driver/mysql v1.4.0
 	github.com/golang/protobuf v1.3.1 // indirect
 	github.com/google/go-cmp v0.3.0 // indirect
 	github.com/gorilla/websocket v1.4.0 // indirect
-	github.com/graph-gophers/dataloader v0.0.0-20180104184831-78139374585c
 	github.com/h2non/filetype v1.0.8
-	github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0
-	github.com/lib/pq v1.1.1 // indirect
-	github.com/mattn/go-sqlite3 v1.10.0 // indirect
 	github.com/minio/minio-go v0.0.0-20180409193742-3d2d02921f05
 	github.com/mitchellh/go-homedir v0.0.0-20161203194507-b8bc1bf76747 // indirect
 	github.com/prometheus/client_golang v0.9.1
@@ -33,7 +28,6 @@ require (
 	golang.org/x/net v0.0.0-20190514140710-3ec191127204 // indirect
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58
 	golang.org/x/text v0.3.0 // indirect
-	google.golang.org/appengine v1.1.0 // indirect
 	gopkg.in/ini.v1 v1.42.0 // indirect
 	gopkg.in/yaml.v2 v2.2.2
 )

go.sum

@@ -21,8 +21,6 @@ github.com/go-ini/ini v1.35.0 h1:D/my3+xOfqZMkJpciRcyqU7XMBUgiZa9qXjZIa8uv2k=
 github.com/go-ini/ini v1.35.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
-github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk=
-github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@@ -40,14 +38,10 @@ github.com/gorilla/mux v1.6.1/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z
 github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
-github.com/graph-gophers/dataloader v0.0.0-20180104184831-78139374585c h1:94S+uoVVMpQAEOrqGjCDyUdML4dJDkh6aC4MYmXECg4=
-github.com/graph-gophers/dataloader v0.0.0-20180104184831-78139374585c/go.mod h1:jk4jk0c5ZISbKaMe8WsVopGB5/15GvGHMdMdPtwlRp4=
 github.com/h2non/filetype v1.0.8 h1:le8gpf+FQA0/DlDABbtisA1KiTS0Xi+YSC/E8yY3Y14=
 github.com/h2non/filetype v1.0.8/go.mod h1:isekKqOuhMj+s/7r3rIeTErIRy4Rub5uBWHfvMusLMU=
 github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0 h1:5B0uxl2lzNRVkJVg+uGHxWtRt4C0Wjc6kJKo5XYx8xE=
-github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
@@ -58,10 +52,6 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4=
-github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
-github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o=
-github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/minio/minio-go v0.0.0-20180409193742-3d2d02921f05 h1:YEyFCqmHcmbcTLqZ7yEAg7VIUat6PuDzGbIctZy888k=
@@ -138,8 +128,6 @@ golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190515012406-7d7faa4812bd h1:oMEQDWVXVNpceQoVd1JN3CQ7LYJJzs5qWqZIUcxXHHw=
 golang.org/x/tools v0.0.0-20190515012406-7d7faa4812bd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs=
-google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=

models/files/db.go

@@ -1,83 +0,0 @@
-package files
-
-import (
-	"crypto/rand"
-	"encoding/binary"
-	"strconv"
-	"time"
-
-	"git.aiterp.net/rpdata/api/internal/store"
-	"git.aiterp.net/rpdata/api/models"
-	"github.com/globalsign/mgo"
-)
-
-var collection *mgo.Collection
-
-func find(query interface{}) (models.File, error) {
-	file := models.File{}
-
-	err := collection.Find(query).One(&file)
-	if err != nil {
-		return models.File{}, err
-	}
-
-	return file, nil
-}
-
-func list(query interface{}) ([]models.File, error) {
-	list := make([]models.File, 0, 32)
-
-	err := collection.Find(query).Sort("-time").All(&list)
-	if err != nil {
-		return nil, err
-	}
-
-	return list, nil
-}
-
-// makeID makes a random file ID that's 32 characters long
-func makeID() string {
-	result := "F" + strconv.FormatInt(time.Now().UnixNano(), 36)
-	offset := 0
-	data := make([]byte, 32)
-
-	rand.Read(data)
-	for len(result) < 32 {
-		result += strconv.FormatUint(binary.LittleEndian.Uint64(data[offset:]), 36)
-		offset += 8
-
-		if offset >= 32 {
-			rand.Read(data)
-			offset = 0
-		}
-	}
-
-	return result[:32]
-}
-
-func init() {
-	store.HandleInit(func(db *mgo.Database) {
-		collection = db.C("file.headers")
-
-		collection.EnsureIndexKey("author")
-		collection.EnsureIndexKey("public")
-		collection.EnsureIndexKey("kind", "name", "author")
-		collection.EnsureIndexKey("author", "public")
-		collection.EnsureIndexKey("kind")
-	})
-}
-
-var allowdMimeTypes = map[string]bool{
-	"":                    false,
-	"image/jpeg":          true,
-	"image/png":           true,
-	"image/gif":           true,
-	"image/tiff":          true,
-	"image/tga":           true,
-	"text/plain":          true,
-	"application/json":    true,
-	"application/pdf":     false,
-	"binary/octet-stream": false,
-	"video/mp4":           false,
-	"audio/mp3":           false,
-}

models/files/find.go

@@ -1,16 +0,0 @@
-package files
-
-import (
-	"git.aiterp.net/rpdata/api/models"
-	"github.com/globalsign/mgo/bson"
-)
-
-// FindID finds a file by ID
-func FindID(id string) (models.File, error) {
-	return find(bson.M{"_id": id})
-}
-
-// FindName finds a file by ID
-func FindName(kind, name, author string) (models.File, error) {
-	return find(bson.M{"name": name, "kind": kind, "author": author})
-}

models/files/insert.go

@@ -1,29 +0,0 @@
-package files
-
-import (
-	"time"
-
-	"git.aiterp.net/rpdata/api/models"
-)
-
-// Insert manually inserts file information into the database. This should never, ever be HTTP API accessible
-func Insert(name, kind, mimeType, author string, time time.Time, size int64, url string) (models.File, error) {
-	file := models.File{
-		ID:       makeID(),
-		Kind:     kind,
-		Time:     time,
-		Public:   false,
-		Author:   author,
-		Name:     name,
-		MimeType: mimeType,
-		Size:     size,
-		URL:      url,
-	}
-
-	err := collection.Insert(file)
-	if err != nil {
-		return models.File{}, err
-	}
-
-	return file, nil
-}

models/files/list.go

@@ -1,36 +0,0 @@
-package files
-
-import (
-	"git.aiterp.net/rpdata/api/models"
-	"github.com/globalsign/mgo/bson"
-)
-
-// Filter for files.List
-type Filter struct {
-	Author   *string
-	Public   *bool
-	MimeType []string
-}
-
-// List lists files according to the standard lookup. By default it's just the author's own files,
-// but if `public` is true it will alos include files made public by other authors. If `mimeTypes` contains
-// any, it will limit the results to that. If `author` is empty, it will only list public files
-func List(filter *Filter) ([]models.File, error) {
-	query := bson.M{}
-
-	if filter != nil {
-		if filter.Author != nil {
-			query["author"] = *filter.Author
-		}
-
-		if filter.Public != nil {
-			query["public"] = *filter.Public
-		}
-
-		if len(filter.MimeType) > 0 {
-			query["mimeTypes"] = bson.M{"$in": filter.MimeType}
-		}
-	}
-
-	return list(query)
-}

models/files/upload.go

@@ -1,52 +0,0 @@
-package files
-
-import (
-	"context"
-	"errors"
-	"io"
-	"time"
-
-	"git.aiterp.net/rpdata/api/internal/store"
-	"git.aiterp.net/rpdata/api/models"
-)
-
-// Upload adds a file to the space.
-func Upload(ctx context.Context, name, mimeType, author string, size int64, input io.Reader) (models.File, error) {
-	if !allowdMimeTypes[mimeType] {
-		return models.File{}, errors.New("File type not allowed:" + mimeType)
-	}
-
-	if name == "" {
-		date := time.Now().UTC().Format("Jan 02 2006 15:04:05 MST")
-		name = "Unnamed file (" + date + ")"
-	}
-	if mimeType == "" {
-		mimeType = "binary/octet-stream"
-	}
-
-	id := makeID()
-
-	path, err := store.UploadFile(ctx, "files", id, mimeType, input, size)
-	if err != nil {
-		return models.File{}, err
-	}
-
-	file := models.File{
-		ID:       id,
-		Kind:     "upload",
-		Time:     time.Now(),
-		Public:   false,
-		Author:   author,
-		Name:     name,
-		MimeType: mimeType,
-		Size:     size,
-		URL:      store.URLFromPath(path),
-	}
-
-	err = collection.Insert(file)
-	if err != nil {
-		return models.File{}, err
-	}
-
-	return file, nil
-}

models/users/db.go

@@ -1,14 +0,0 @@
-package users
-
-import (
-	"git.aiterp.net/rpdata/api/internal/store"
-	"github.com/globalsign/mgo"
-)
-
-var collection *mgo.Collection
-
-func init() {
-	store.HandleInit(func(db *mgo.Database) {
-		collection = db.C("core.users")
-	})
-}

models/users/ensure.go

@@ -1,34 +0,0 @@
-package users
-
-import (
-	"git.aiterp.net/rpdata/api/models"
-	"github.com/globalsign/mgo"
-)
-
-// Ensure finds a user by id, or makes a new one.
-func Ensure(id string) (models.User, error) {
-	user := models.User{}
-	err := collection.FindId(id).One(&user)
-
-	if err == mgo.ErrNotFound {
-		user = models.User{
-			ID:   id,
-			Nick: "",
-			Permissions: []string{
-				"member",
-				"log.edit",
-				"post.edit",
-				"post.move",
-				"post.remove",
-				"file.upload",
-			},
-		}
-
-		err := collection.Insert(user)
-		if err != nil {
-			return models.User{}, err
-		}
-	}
-
-	return user, err
-}

models/users/find.go

@@ -1,13 +0,0 @@
-package users
-
-import (
-	"git.aiterp.net/rpdata/api/models"
-)
-
-// Find finds a user by id
-func Find(id string) (models.User, error) {
-	user := models.User{}
-	err := collection.FindId(id).One(&user)
-
-	return user, err
-}