diff --git a/Dockerfile b/Dockerfile index 9f75369..647e51b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,11 +12,6 @@ RUN go generate ./... # Build server and tools (CGO disabled to make them 100% static) ENV CGO_ENABLED 0 RUN go build -installsuffix cgo -ldflags="-s -w" -o /binaries/rpdata-server ./cmd/rpdata-server -RUN go build -installsuffix cgo -ldflags="-s -w" -o /binaries/rpdata-lb2charimport ./cmd/rpdata-lb2charimport -RUN go build -installsuffix cgo -ldflags="-s -w" -o /binaries/rpdata-lb2logimport ./cmd/rpdata-lb2logimport -# TODO: RUN go build -installsuffix cgo -ldflags="-s -w" -o /binaries/rpdata-wikifileimport ./cmd/rpdata-wikifileimport -RUN go build -installsuffix cgo -ldflags="-s -w" -o /binaries/rpdata-ensurechannels ./cmd/rpdata-ensurechannels -RUN go build -installsuffix cgo -ldflags="-s -w" -o /binaries/rpdata-as2storyimport ./cmd/rpdata-as2storyimport ## 2. Distribute # Use alpine linux diff --git a/cmd/rpdata-as2storyimport/main.go b/cmd/rpdata-as2storyimport/main.go deleted file mode 100644 index 911feda..0000000 --- a/cmd/rpdata-as2storyimport/main.go +++ /dev/null @@ -1,139 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "log" - "strings" - "time" - - "git.aiterp.net/rpdata/api/internal/store" - "git.aiterp.net/rpdata/api/models" - "git.aiterp.net/rpdata/api/models/chapters" - "git.aiterp.net/rpdata/api/models/stories" - _ "github.com/go-sql-driver/mysql" - "github.com/jmoiron/sqlx" -) - -var flagHost = flag.String("host", "127.0.0.1:3306", "SQL host") -var flagDB = flag.String("database", "aitestory", "SQL database") -var flagUser = flag.String("user", "aitestory", "SQL user") -var flagPassword = flag.String("password", "", "SQL password") - -func main() { - flag.Parse() - - db, err := sqlx.Connect("mysql", fmt.Sprintf("%s:%s@(%s)/%s", *flagUser, *flagPassword, *flagHost, *flagDB)) - if err != nil { - log.Fatalln(err) - } - - err = db.Ping() - if err != nil { - log.Fatalln(err) - } - - store.Init() - - results := make([]storyResult, 0, 64) - rows, err := db.Queryx("SELECT * FROM page WHERE unlisted=0;") - for rows.Next() { - result := storyResult{} - err := rows.StructScan(&result) - if err != nil { - log.Fatalln(err) - } - results = append(results, result) - } - - tagResults := make([]tagResult, 0, 256) - rows, err = db.Queryx("SELECT page_id,type,name FROM page_tag LEFT JOIN tag ON tag_id=tag.id;") - for rows.Next() { - result := tagResult{} - err := rows.StructScan(&result) - if err != nil { - log.Fatalln(err) - } - tagResults = append(tagResults, result) - } - - for _, result := range results { - fictionalDate, err := time.Parse("2006-01-02 15:04:05", result.FictionalDate) - if err != nil { - if result.FictionalDate != "0000-00-00 00:00:00" { - log.Fatalln(err) - } - } - if fictionalDate.Year() < 1800 { - fictionalDate = time.Time{} - } - - publishDate, err := time.Parse("2006-01-02 15:04:05", result.PublishDate) - if err != nil { - log.Fatalln(err) - } - - tags := make([]models.Tag, 0, 8) - for _, tagResult := range tagResults { - if tagResult.PageID == result.ID { - tags = append(tags, models.Tag{Kind: models.TagKind(tagResult.Type), Name: tagResult.Name}) - } - } - - category := models.StoryCategory(result.Category) - if !category.IsValid() { - log.Println(result.Name, "does not have a valid category:", result.Category) - continue - } - - story, err := stories.Add(result.Name, result.Author[5:], category, true, false, tags, publishDate, fictionalDate) - if err != nil { - log.Fatalln(err) - } - - // Change the story title - title := result.Name - if strings.HasPrefix(result.Source, "#") { - firstNewline := strings.Index(result.Source, "\n") - result.Name = strings.Replace(result.Source[1:firstNewline], "\r", "", -1) - result.Source = result.Source[firstNewline+1:] - - result.Source = strings.Replace(result.Source, "\r\n", "\n", -1) - if strings.HasPrefix(result.Source, "\r") || strings.HasPrefix(result.Source, "\n") { - result.Source = result.Source[1:] - } - } - - chapter, err := chapters.Add(story, title, result.Author[5:], result.Source, publishDate, &fictionalDate, models.ChapterCommentModeDisabled) - if err != nil { - log.Fatalln(err) - } - - fmt.Println(result.ID, "->", story.ID, chapter.ID) - } -} - -type tagResult struct { - PageID string `db:"page_id"` - Type string `db:"type"` - Name string `db:"name"` -} - -type storyResult struct { - ID string `db:"id"` - Name string `db:"name"` - Author string `db:"author"` - Category string `db:"category"` - FictionalDate string `db:"fictional_date"` - PublishDate string `db:"publish_date"` - EditDate string `db:"edit_date"` - Unlisted bool `db:"unlisted"` - Dated bool `db:"dated"` - Spesific bool `db:"specific"` - Indexed bool `db:"indexed"` - Published bool `db:"published"` - Type string `db:"type"` - Source string `db:"source"` - Cache string `db:"cache"` - BackgroundURL *string `db:"background_url"` -} diff --git a/cmd/rpdata-ensurechannels/main.go b/cmd/rpdata-ensurechannels/main.go deleted file mode 100644 index 5a9b41b..0000000 --- a/cmd/rpdata-ensurechannels/main.go +++ /dev/null @@ -1,41 +0,0 @@ -package main - -import ( - "fmt" - "os" - - "git.aiterp.net/rpdata/api/internal/store" - "git.aiterp.net/rpdata/api/models/channels" - "git.aiterp.net/rpdata/api/models/logs" -) - -func main() { - err := store.Init() - if err != nil { - fmt.Fprintln(os.Stderr, err) - return - } - - logs, err := logs.List(nil) - if err != nil { - fmt.Fprintln(os.Stderr, err) - return - } - - added := make(map[string]bool, 1024) - for _, log := range logs { - if added[log.ChannelName] { - continue - } - - _, err := channels.Ensure(log.ChannelName, false) - if err != nil { - fmt.Fprintln(os.Stderr, log.ID, err) - continue - } - - added[log.ChannelName] = true - - fmt.Println(log.ChannelName, "ensured") - } -} diff --git a/cmd/rpdata-lb2charimport/.gitignore b/cmd/rpdata-lb2charimport/.gitignore deleted file mode 100644 index 9aaf1e6..0000000 --- a/cmd/rpdata-lb2charimport/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -characters.json -characters.cson -debug \ No newline at end of file diff --git a/cmd/rpdata-lb2charimport/data.go b/cmd/rpdata-lb2charimport/data.go deleted file mode 100644 index cf75a6b..0000000 --- a/cmd/rpdata-lb2charimport/data.go +++ /dev/null @@ -1,55 +0,0 @@ -package main - -import ( - "encoding/json" - "io" - "strings" -) - -type charInfo struct { - Nicks []string `json:"nicks"` - Name string `json:"name"` - Author string `json:"player"` - ShortName string `json:"first"` -} - -func load(reader io.Reader) ([]charInfo, error) { - data := make(map[string]interface{}) - err := json.NewDecoder(reader).Decode(&data) - if err != nil { - return nil, err - } - - links := make(map[string]string, len(data)) - infos := make([]charInfo, 0, 64) - - for key, value := range data { - if info, ok := value.(map[string]interface{}); ok { - name := info["name"].(string) - author := info["player"].(string) - shortName, ok := info["first"].(string) - if !ok { - shortName = strings.SplitN(name, " ", 2)[0] - } - - infos = append(infos, charInfo{ - Nicks: []string{key}, - Name: name, - Author: author, - ShortName: shortName, - }) - } else if nick, ok := value.(string); ok { - links[key] = nick - } - } - - for key, value := range links { - for i := range infos { - if infos[i].Nicks[0] == value { - infos[i].Nicks = append(infos[i].Nicks, key) - } - } - } - - return infos, nil -} diff --git a/cmd/rpdata-lb2charimport/main.go b/cmd/rpdata-lb2charimport/main.go deleted file mode 100644 index c276057..0000000 --- a/cmd/rpdata-lb2charimport/main.go +++ /dev/null @@ -1,62 +0,0 @@ -package main - -import ( - "flag" - "log" - "os" - - "git.aiterp.net/rpdata/api/internal/store" - "git.aiterp.net/rpdata/api/models/characters" -) - -var fileName = flag.String("file", "./characters.json", "json file to load") - -func main() { - file, err := os.Open(*fileName) - if err != nil { - log.Fatalln("Open:", err) - } - - infos, err := load(file) - if err != nil { - log.Fatalln("Parse:", err) - } - - err = store.Init() - if err != nil { - log.Fatalln("Store:", err) - } - - charsAdded, charsFailed := 0, 0 - nicksAdded, nicksFailed := 0, 0 - - for _, info := range infos { - char, err := characters.Add(info.Nicks[0], info.Name, info.ShortName, info.Author, "") - if err != nil { - log.Println(info.Nicks[0], "failed to insert:", err) - charsFailed++ - - char, err = characters.FindNick(info.Nicks[0]) - if err != nil { - continue - } - } else { - log.Println(info.Nicks[0], "added") - charsAdded++ - } - - for _, alt := range info.Nicks[1:] { - char, err = characters.AddNick(char, alt) - if err != nil { - log.Println(info.Nicks[0], "failed to add nick", alt, "error:", err) - nicksFailed++ - } else { - log.Println(info.Nicks[0], "addded nick", alt) - nicksAdded++ - } - } - } - - log.Printf("Characters – %d/%d", charsAdded, charsFailed+charsAdded) - log.Printf("Alt. Nicks – %d/%d", nicksAdded, nicksFailed+nicksAdded) -} diff --git a/cmd/rpdata-lb2logimport/.gitignore b/cmd/rpdata-lb2logimport/.gitignore deleted file mode 100644 index 2d164fe..0000000 --- a/cmd/rpdata-lb2logimport/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -logs -debug \ No newline at end of file diff --git a/cmd/rpdata-lb2logimport/line.go b/cmd/rpdata-lb2logimport/line.go deleted file mode 100644 index f352a4c..0000000 --- a/cmd/rpdata-lb2logimport/line.go +++ /dev/null @@ -1,85 +0,0 @@ -package main - -import ( - "bufio" - "errors" - "io" - "strconv" - "strings" - "time" -) - -type logLine struct { - Verb string - Args []string - Text string -} - -func parseFile(reader io.Reader) []logLine { - bufReader := bufio.NewReader(reader) - results := make([]logLine, 0, 512) - - for { - line, err := bufReader.ReadString('\n') - if err == io.EOF { - if len(line) < 2 { - break - } - } else if err != nil { - break - } - - if len(line) <= 2 { - continue - } - - line = strings.Replace(line, "\n", "", 1) - line = strings.Replace(line, "\r", "", 1) - - results = append(results, parseLine(line)) - } - - return results -} - -func parseLine(line string) logLine { - textSplit := strings.SplitN(line, " :", 2) - tokens := strings.Split(textSplit[0], " ") - - ll := logLine{ - Verb: tokens[0], - } - - if len(tokens) > 1 { - ll.Args = tokens[1:] - } - - if len(textSplit) > 1 { - ll.Text = textSplit[1] - } - - return ll -} - -func parseFilename(fname string) (date time.Time, channel string, err error) { - date, err = time.ParseInLocation("2006-01-02_150405", fname[:17], time.Local) - if err != nil { - return - } - - ms, err := strconv.Atoi(fname[17:20]) - if err != nil { - return - } - - date = date.Add(time.Duration(ms) * time.Millisecond) - - if len(fname) < 23 { - err = errors.New("filename too short") - return - } - - channel = fname[21:] - - return -} diff --git a/cmd/rpdata-lb2logimport/main.go b/cmd/rpdata-lb2logimport/main.go deleted file mode 100644 index 56fc702..0000000 --- a/cmd/rpdata-lb2logimport/main.go +++ /dev/null @@ -1,110 +0,0 @@ -package main - -import ( - "fmt" - "log" - "os" - "path" - "strings" - "time" - - "git.aiterp.net/rpdata/api/models/posts" - - "git.aiterp.net/rpdata/api/internal/store" - "git.aiterp.net/rpdata/api/models/logs" -) - -var prefixReplacer = strings.NewReplacer("+", "", "@", "", "!", "", "%", "") - -func main() { - err := store.Init() - if err != nil { - log.Fatalln(err) - } - - for _, filepath := range os.Args[1:] { - name := strings.Replace(path.Base(filepath), ".txt", "", 1) - file, err := os.Open(filepath) - if err != nil { - log.Println(filepath, err) - continue - } - logLines := parseFile(file) - file.Close() - - // Get title and event - title := "" - event := "" - for _, line := range logLines { - if line.Verb == "TITLE" { - title = line.Text - } - if line.Verb == "TAG" { - event = line.Text - } - } - - date, channel, err := parseFilename(name) - if err != nil { - log.Fatalln(err) - } - - l, err := logs.Add(date, channel, title, event, "", false) - if err != nil { - log.Println(err) - continue - } - - _, err = posts.Add(l, time.Now(), "annotation.info", "rpdata-lb2logimport", "This logfile is imported from aitelogs2 and may contain errors or wrong timestamps.") - if err != nil { - log.Println(err) - } - - for _, line := range logLines { - if line.Verb != "CHARS" { - continue - } - - _, err = posts.Add(l, l.Date, "chars", prefixReplacer.Replace(line.Args[0]), line.Text) - if err != nil { - log.Println(err) - } - } - - for _, line := range logLines { - if line.Verb != "SCENE" && line.Verb != "ACTION" && line.Verb != "TEXT" { - continue - } - - postTime, err := time.ParseInLocation("2006-01-02 15:04:05", date.Format("2006-01-02")+" "+line.Args[1], time.Local) - diff := postTime.Sub(date) - if err != nil { - log.Println(err) - continue - } - if diff < 0 { - if diff > -time.Second { - postTime = postTime.Add(diff) - } else { - postTime = postTime.Add(time.Hour * 24) - } - } - - if line.Args[0][0] == '=' { - line.Verb = "SCENE" - } - - _, err = posts.Add(l, postTime, strings.ToLower(line.Verb), prefixReplacer.Replace(line.Args[0]), line.Text) - if err != nil { - log.Println(err) - } - } - - l, err = logs.UpdateCharacters(l, nil) - if err != nil { - log.Println(err) - } - - fmt.Println(l.ID, "completed") - } -} diff --git a/cmd/rpdata-wikifileimport/main.go b/cmd/rpdata-wikifileimport/main.go deleted file mode 100644 index 9274338..0000000 --- a/cmd/rpdata-wikifileimport/main.go +++ /dev/null @@ -1,77 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "log" - "net/http" - "time" - - "git.aiterp.net/rpdata/api/internal/config" - "git.aiterp.net/rpdata/api/internal/store" - "git.aiterp.net/rpdata/api/models/files" -) - -func main() { - client := http.Client{Timeout: time.Second * 30} - - res, err := client.Get(config.Global().Wiki.URL + "?action=query&list=allimages&ailimit=5&aiprop=dimensions|mime|url|user|timestamp&format=json&ailimit=500") - if err != nil { - log.Fatalln(err) - } - - if res.StatusCode != 200 { - log.Fatalln(res.Status) - } - - data := ResponseData{} - err = json.NewDecoder(res.Body).Decode(&data) - if err != nil { - log.Fatalln(err) - } - - store.Init() - - for _, info := range data.Query.Allimages { - existing, err := files.FindName("wiki", info.Name, info.User) - if err == nil && existing.ID != "" { - log.Println("Skipping", info.Name, "because it already exists") - continue - } - - ts, err := time.Parse(time.RFC3339, info.Timestamp) - if err != nil { - log.Println("Skipping", info.Name, "error:", err) - continue - } - - file, err := files.Insert(info.Name, "wiki", info.Mime, info.User, ts, info.Size, info.URL) - if err != nil { - log.Println("Skipping", info.Name, "error:", err) - continue - } - - fmt.Println(file.Name, "inserted ( id:", file.ID, ")") - } -} - -// The ResponseData from the wiki looks like this -type ResponseData struct { - Batchcomplete string `json:"batchcomplete"` - Query struct { - Allimages []struct { - Descriptionshorturl string `json:"descriptionshorturl"` - Descriptionurl string `json:"descriptionurl"` - Height int `json:"height"` - Mime string `json:"mime"` - Name string `json:"name"` - Ns int `json:"ns"` - Size int64 `json:"size"` - Timestamp string `json:"timestamp"` - Title string `json:"title"` - URL string `json:"url"` - User string `json:"user"` - Width int `json:"width"` - } `json:"allimages"` - } `json:"query"` -} diff --git a/go.mod b/go.mod index a6d2f80..894ab7a 100644 --- a/go.mod +++ b/go.mod @@ -8,15 +8,10 @@ require ( github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/globalsign/mgo v0.0.0-20180403085842-f76e4f9da92e github.com/go-ini/ini v1.35.0 // indirect - github.com/go-sql-driver/mysql v1.4.0 github.com/golang/protobuf v1.3.1 // indirect github.com/google/go-cmp v0.3.0 // indirect github.com/gorilla/websocket v1.4.0 // indirect - github.com/graph-gophers/dataloader v0.0.0-20180104184831-78139374585c github.com/h2non/filetype v1.0.8 - github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0 - github.com/lib/pq v1.1.1 // indirect - github.com/mattn/go-sqlite3 v1.10.0 // indirect github.com/minio/minio-go v0.0.0-20180409193742-3d2d02921f05 github.com/mitchellh/go-homedir v0.0.0-20161203194507-b8bc1bf76747 // indirect github.com/prometheus/client_golang v0.9.1 @@ -33,7 +28,6 @@ require ( golang.org/x/net v0.0.0-20190514140710-3ec191127204 // indirect golang.org/x/sync v0.0.0-20190423024810-112230192c58 golang.org/x/text v0.3.0 // indirect - google.golang.org/appengine v1.1.0 // indirect gopkg.in/ini.v1 v1.42.0 // indirect gopkg.in/yaml.v2 v2.2.2 ) diff --git a/go.sum b/go.sum index 0383972..aef6733 100644 --- a/go.sum +++ b/go.sum @@ -21,8 +21,6 @@ github.com/go-ini/ini v1.35.0 h1:D/my3+xOfqZMkJpciRcyqU7XMBUgiZa9qXjZIa8uv2k= github.com/go-ini/ini v1.35.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk= -github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -40,14 +38,10 @@ github.com/gorilla/mux v1.6.1/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/graph-gophers/dataloader v0.0.0-20180104184831-78139374585c h1:94S+uoVVMpQAEOrqGjCDyUdML4dJDkh6aC4MYmXECg4= -github.com/graph-gophers/dataloader v0.0.0-20180104184831-78139374585c/go.mod h1:jk4jk0c5ZISbKaMe8WsVopGB5/15GvGHMdMdPtwlRp4= github.com/h2non/filetype v1.0.8 h1:le8gpf+FQA0/DlDABbtisA1KiTS0Xi+YSC/E8yY3Y14= github.com/h2non/filetype v1.0.8/go.mod h1:isekKqOuhMj+s/7r3rIeTErIRy4Rub5uBWHfvMusLMU= github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0 h1:5B0uxl2lzNRVkJVg+uGHxWtRt4C0Wjc6kJKo5XYx8xE= -github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= @@ -58,10 +52,6 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4= -github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o= -github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/minio/minio-go v0.0.0-20180409193742-3d2d02921f05 h1:YEyFCqmHcmbcTLqZ7yEAg7VIUat6PuDzGbIctZy888k= @@ -138,8 +128,6 @@ golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190515012406-7d7faa4812bd h1:oMEQDWVXVNpceQoVd1JN3CQ7LYJJzs5qWqZIUcxXHHw= golang.org/x/tools v0.0.0-20190515012406-7d7faa4812bd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= diff --git a/models/files/db.go b/models/files/db.go deleted file mode 100644 index a36f392..0000000 --- a/models/files/db.go +++ /dev/null @@ -1,83 +0,0 @@ -package files - -import ( - "crypto/rand" - "encoding/binary" - "strconv" - "time" - - "git.aiterp.net/rpdata/api/internal/store" - "git.aiterp.net/rpdata/api/models" - "github.com/globalsign/mgo" -) - -var collection *mgo.Collection - -func find(query interface{}) (models.File, error) { - file := models.File{} - - err := collection.Find(query).One(&file) - if err != nil { - return models.File{}, err - } - - return file, nil -} - -func list(query interface{}) ([]models.File, error) { - list := make([]models.File, 0, 32) - - err := collection.Find(query).Sort("-time").All(&list) - if err != nil { - return nil, err - } - - return list, nil -} - -// makeID makes a random file ID that's 32 characters long -func makeID() string { - result := "F" + strconv.FormatInt(time.Now().UnixNano(), 36) - offset := 0 - data := make([]byte, 32) - - rand.Read(data) - for len(result) < 32 { - result += strconv.FormatUint(binary.LittleEndian.Uint64(data[offset:]), 36) - offset += 8 - - if offset >= 32 { - rand.Read(data) - offset = 0 - } - } - - return result[:32] -} - -func init() { - store.HandleInit(func(db *mgo.Database) { - collection = db.C("file.headers") - - collection.EnsureIndexKey("author") - collection.EnsureIndexKey("public") - collection.EnsureIndexKey("kind", "name", "author") - collection.EnsureIndexKey("author", "public") - collection.EnsureIndexKey("kind") - }) -} - -var allowdMimeTypes = map[string]bool{ - "": false, - "image/jpeg": true, - "image/png": true, - "image/gif": true, - "image/tiff": true, - "image/tga": true, - "text/plain": true, - "application/json": true, - "application/pdf": false, - "binary/octet-stream": false, - "video/mp4": false, - "audio/mp3": false, -} diff --git a/models/files/find.go b/models/files/find.go deleted file mode 100644 index de14caf..0000000 --- a/models/files/find.go +++ /dev/null @@ -1,16 +0,0 @@ -package files - -import ( - "git.aiterp.net/rpdata/api/models" - "github.com/globalsign/mgo/bson" -) - -// FindID finds a file by ID -func FindID(id string) (models.File, error) { - return find(bson.M{"_id": id}) -} - -// FindName finds a file by ID -func FindName(kind, name, author string) (models.File, error) { - return find(bson.M{"name": name, "kind": kind, "author": author}) -} diff --git a/models/files/insert.go b/models/files/insert.go deleted file mode 100644 index a943912..0000000 --- a/models/files/insert.go +++ /dev/null @@ -1,29 +0,0 @@ -package files - -import ( - "time" - - "git.aiterp.net/rpdata/api/models" -) - -// Insert manually inserts file information into the database. This should never, ever be HTTP API accessible -func Insert(name, kind, mimeType, author string, time time.Time, size int64, url string) (models.File, error) { - file := models.File{ - ID: makeID(), - Kind: kind, - Time: time, - Public: false, - Author: author, - Name: name, - MimeType: mimeType, - Size: size, - URL: url, - } - - err := collection.Insert(file) - if err != nil { - return models.File{}, err - } - - return file, nil -} diff --git a/models/files/list.go b/models/files/list.go deleted file mode 100644 index aa5156c..0000000 --- a/models/files/list.go +++ /dev/null @@ -1,36 +0,0 @@ -package files - -import ( - "git.aiterp.net/rpdata/api/models" - "github.com/globalsign/mgo/bson" -) - -// Filter for files.List -type Filter struct { - Author *string - Public *bool - MimeType []string -} - -// List lists files according to the standard lookup. By default it's just the author's own files, -// but if `public` is true it will alos include files made public by other authors. If `mimeTypes` contains -// any, it will limit the results to that. If `author` is empty, it will only list public files -func List(filter *Filter) ([]models.File, error) { - query := bson.M{} - - if filter != nil { - if filter.Author != nil { - query["author"] = *filter.Author - } - - if filter.Public != nil { - query["public"] = *filter.Public - } - - if len(filter.MimeType) > 0 { - query["mimeTypes"] = bson.M{"$in": filter.MimeType} - } - } - - return list(query) -} diff --git a/models/files/upload.go b/models/files/upload.go deleted file mode 100644 index e578959..0000000 --- a/models/files/upload.go +++ /dev/null @@ -1,52 +0,0 @@ -package files - -import ( - "context" - "errors" - "io" - "time" - - "git.aiterp.net/rpdata/api/internal/store" - "git.aiterp.net/rpdata/api/models" -) - -// Upload adds a file to the space. -func Upload(ctx context.Context, name, mimeType, author string, size int64, input io.Reader) (models.File, error) { - if !allowdMimeTypes[mimeType] { - return models.File{}, errors.New("File type not allowed:" + mimeType) - } - - if name == "" { - date := time.Now().UTC().Format("Jan 02 2006 15:04:05 MST") - name = "Unnamed file (" + date + ")" - } - if mimeType == "" { - mimeType = "binary/octet-stream" - } - - id := makeID() - - path, err := store.UploadFile(ctx, "files", id, mimeType, input, size) - if err != nil { - return models.File{}, err - } - - file := models.File{ - ID: id, - Kind: "upload", - Time: time.Now(), - Public: false, - Author: author, - Name: name, - MimeType: mimeType, - Size: size, - URL: store.URLFromPath(path), - } - - err = collection.Insert(file) - if err != nil { - return models.File{}, err - } - - return file, nil -} diff --git a/models/users/db.go b/models/users/db.go deleted file mode 100644 index dca3310..0000000 --- a/models/users/db.go +++ /dev/null @@ -1,14 +0,0 @@ -package users - -import ( - "git.aiterp.net/rpdata/api/internal/store" - "github.com/globalsign/mgo" -) - -var collection *mgo.Collection - -func init() { - store.HandleInit(func(db *mgo.Database) { - collection = db.C("core.users") - }) -} diff --git a/models/users/ensure.go b/models/users/ensure.go deleted file mode 100644 index 2e03912..0000000 --- a/models/users/ensure.go +++ /dev/null @@ -1,34 +0,0 @@ -package users - -import ( - "git.aiterp.net/rpdata/api/models" - "github.com/globalsign/mgo" -) - -// Ensure finds a user by id, or makes a new one. -func Ensure(id string) (models.User, error) { - user := models.User{} - err := collection.FindId(id).One(&user) - - if err == mgo.ErrNotFound { - user = models.User{ - ID: id, - Nick: "", - Permissions: []string{ - "member", - "log.edit", - "post.edit", - "post.move", - "post.remove", - "file.upload", - }, - } - - err := collection.Insert(user) - if err != nil { - return models.User{}, err - } - } - - return user, err -} diff --git a/models/users/find.go b/models/users/find.go deleted file mode 100644 index a6fd5c4..0000000 --- a/models/users/find.go +++ /dev/null @@ -1,13 +0,0 @@ -package users - -import ( - "git.aiterp.net/rpdata/api/models" -) - -// Find finds a user by id -func Find(id string) (models.User, error) { - user := models.User{} - err := collection.FindId(id).One(&user) - - return user, err -}