rpdata/api
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

More destruction. Utils too, since they cause weird go mod shenanigans.
the build was successful
Details

Browse Source
This commit is contained in:
gisle
2019-08-26 22:22:05 +02:00
parent e690e7ab97
commit 8a313b3bb2
20 changed files with 0 additions and 874 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Dockerfile
-5
View File
@@ -12,11 +12,6 @@ RUN go generate ./...
# Build server and tools (CGO disabled to make them 100% static)
ENV CGO_ENABLED 0
RUN go build -installsuffix cgo -ldflags="-s -w" -o /binaries/rpdata-server ./cmd/rpdata-server
RUN go build -installsuffix cgo -ldflags="-s -w" -o /binaries/rpdata-lb2charimport ./cmd/rpdata-lb2charimport
RUN go build -installsuffix cgo -ldflags="-s -w" -o /binaries/rpdata-lb2logimport ./cmd/rpdata-lb2logimport
# TODO: RUN go build -installsuffix cgo -ldflags="-s -w" -o /binaries/rpdata-wikifileimport ./cmd/rpdata-wikifileimport
RUN go build -installsuffix cgo -ldflags="-s -w" -o /binaries/rpdata-ensurechannels ./cmd/rpdata-ensurechannels
RUN go build -installsuffix cgo -ldflags="-s -w" -o /binaries/rpdata-as2storyimport ./cmd/rpdata-as2storyimport
## 2. Distribute
# Use alpine linux
cmd/rpdata-as2storyimport/main.go
-139
View File
@@ -1,139 +0,0 @@
package main
import (
"flag"
"fmt"
"log"
"strings"
"time"
"git.aiterp.net/rpdata/api/internal/store"
"git.aiterp.net/rpdata/api/models"
"git.aiterp.net/rpdata/api/models/chapters"
"git.aiterp.net/rpdata/api/models/stories"
_ "github.com/go-sql-driver/mysql"
"github.com/jmoiron/sqlx"
)
var flagHost = flag.String("host", "127.0.0.1:3306", "SQL host")
var flagDB = flag.String("database", "aitestory", "SQL database")
var flagUser = flag.String("user", "aitestory", "SQL user")
var flagPassword = flag.String("password", "", "SQL password")
func main() {
flag.Parse()
db, err := sqlx.Connect("mysql", fmt.Sprintf("%s:%s@(%s)/%s", *flagUser, *flagPassword, *flagHost, *flagDB))
if err != nil {
log.Fatalln(err)
}
err = db.Ping()
if err != nil {
log.Fatalln(err)
}
store.Init()
results := make([]storyResult, 0, 64)
rows, err := db.Queryx("SELECT * FROM page WHERE unlisted=0;")
for rows.Next() {
result := storyResult{}
err := rows.StructScan(&result)
if err != nil {
log.Fatalln(err)
}
results = append(results, result)
}
tagResults := make([]tagResult, 0, 256)
rows, err = db.Queryx("SELECT page_id,type,name FROM page_tag LEFT JOIN tag ON tag_id=tag.id;")
for rows.Next() {
result := tagResult{}
err := rows.StructScan(&result)
if err != nil {
log.Fatalln(err)
}
tagResults = append(tagResults, result)
}
for _, result := range results {
fictionalDate, err := time.Parse("2006-01-02 15:04:05", result.FictionalDate)
if err != nil {
if result.FictionalDate != "0000-00-00 00:00:00" {
log.Fatalln(err)
}
}
if fictionalDate.Year() < 1800 {
fictionalDate = time.Time{}
}
publishDate, err := time.Parse("2006-01-02 15:04:05", result.PublishDate)
if err != nil {
log.Fatalln(err)
}
tags := make([]models.Tag, 0, 8)
for _, tagResult := range tagResults {
if tagResult.PageID == result.ID {
tags = append(tags, models.Tag{Kind: models.TagKind(tagResult.Type), Name: tagResult.Name})
}
}
category := models.StoryCategory(result.Category)
if !category.IsValid() {
log.Println(result.Name, "does not have a valid category:", result.Category)
continue
}
story, err := stories.Add(result.Name, result.Author[5:], category, true, false, tags, publishDate, fictionalDate)
if err != nil {
log.Fatalln(err)
}
// Change the story title
title := result.Name
if strings.HasPrefix(result.Source, "#") {
firstNewline := strings.Index(result.Source, "\n")
result.Name = strings.Replace(result.Source[1:firstNewline], "\r", "", -1)
result.Source = result.Source[firstNewline+1:]
result.Source = strings.Replace(result.Source, "\r\n", "\n", -1)
if strings.HasPrefix(result.Source, "\r") || strings.HasPrefix(result.Source, "\n") {
result.Source = result.Source[1:]
}
}
chapter, err := chapters.Add(story, title, result.Author[5:], result.Source, publishDate, &fictionalDate, models.ChapterCommentModeDisabled)
if err != nil {
log.Fatalln(err)
}
fmt.Println(result.ID, "->", story.ID, chapter.ID)
}
}
type tagResult struct {
PageID string `db:"page_id"`
Type string `db:"type"`
Name string `db:"name"`
}
type storyResult struct {
ID string `db:"id"`
Name string `db:"name"`
Author string `db:"author"`
Category string `db:"category"`
FictionalDate string `db:"fictional_date"`
PublishDate string `db:"publish_date"`
EditDate string `db:"edit_date"`
Unlisted bool `db:"unlisted"`
Dated bool `db:"dated"`
Spesific bool `db:"specific"`
Indexed bool `db:"indexed"`
Published bool `db:"published"`
Type string `db:"type"`
Source string `db:"source"`
Cache string `db:"cache"`
BackgroundURL *string `db:"background_url"`
}
cmd/rpdata-ensurechannels/main.go
-41
View File
@@ -1,41 +0,0 @@
package main
import (
"fmt"
"os"
"git.aiterp.net/rpdata/api/internal/store"
"git.aiterp.net/rpdata/api/models/channels"
"git.aiterp.net/rpdata/api/models/logs"
)
func main() {
err := store.Init()
if err != nil {
fmt.Fprintln(os.Stderr, err)
return
}
logs, err := logs.List(nil)
if err != nil {
fmt.Fprintln(os.Stderr, err)
return
}
added := make(map[string]bool, 1024)
for _, log := range logs {
if added[log.ChannelName] {
continue
}
_, err := channels.Ensure(log.ChannelName, false)
if err != nil {
fmt.Fprintln(os.Stderr, log.ID, err)
continue
}
added[log.ChannelName] = true
fmt.Println(log.ChannelName, "ensured")
}
}
cmd/rpdata-lb2charimport/.gitignore
-3
View File
@@ -1,3 +0,0 @@
characters.json
characters.cson
debug
cmd/rpdata-lb2charimport/data.go
-55
View File
@@ -1,55 +0,0 @@
package main
import (
"encoding/json"
"io"
"strings"
)
type charInfo struct {
Nicks []string `json:"nicks"`
Name string `json:"name"`
Author string `json:"player"`
ShortName string `json:"first"`
}
func load(reader io.Reader) ([]charInfo, error) {
data := make(map[string]interface{})
err := json.NewDecoder(reader).Decode(&data)
if err != nil {
return nil, err
}
links := make(map[string]string, len(data))
infos := make([]charInfo, 0, 64)
for key, value := range data {
if info, ok := value.(map[string]interface{}); ok {
name := info["name"].(string)
author := info["player"].(string)
shortName, ok := info["first"].(string)
if !ok {
shortName = strings.SplitN(name, " ", 2)[0]
}
infos = append(infos, charInfo{
Nicks: []string{key},
Name: name,
Author: author,
ShortName: shortName,
})
} else if nick, ok := value.(string); ok {
links[key] = nick
}
}
for key, value := range links {
for i := range infos {
if infos[i].Nicks[0] == value {
infos[i].Nicks = append(infos[i].Nicks, key)
}
}
}
return infos, nil
}
cmd/rpdata-lb2charimport/main.go
-62
View File
@@ -1,62 +0,0 @@
package main
import (
"flag"
"log"
"os"
"git.aiterp.net/rpdata/api/internal/store"
"git.aiterp.net/rpdata/api/models/characters"
)
var fileName = flag.String("file", "./characters.json", "json file to load")
func main() {
file, err := os.Open(*fileName)
if err != nil {
log.Fatalln("Open:", err)
}
infos, err := load(file)
if err != nil {
log.Fatalln("Parse:", err)
}
err = store.Init()
if err != nil {
log.Fatalln("Store:", err)
}
charsAdded, charsFailed := 0, 0
nicksAdded, nicksFailed := 0, 0
for _, info := range infos {
char, err := characters.Add(info.Nicks[0], info.Name, info.ShortName, info.Author, "")
if err != nil {
log.Println(info.Nicks[0], "failed to insert:", err)
charsFailed++
char, err = characters.FindNick(info.Nicks[0])
if err != nil {
continue
}
} else {
log.Println(info.Nicks[0], "added")
charsAdded++
}
for _, alt := range info.Nicks[1:] {
char, err = characters.AddNick(char, alt)
if err != nil {
log.Println(info.Nicks[0], "failed to add nick", alt, "error:", err)
nicksFailed++
} else {
log.Println(info.Nicks[0], "addded nick", alt)
nicksAdded++
}
}
}
log.Printf("Characters – %d/%d", charsAdded, charsFailed+charsAdded)
log.Printf("Alt. Nicks – %d/%d", nicksAdded, nicksFailed+nicksAdded)
}
cmd/rpdata-lb2logimport/.gitignore
-2
View File
@@ -1,2 +0,0 @@
logs
debug
cmd/rpdata-lb2logimport/line.go
-85
View File
@@ -1,85 +0,0 @@
package main
import (
"bufio"
"errors"
"io"
"strconv"
"strings"
"time"
)
type logLine struct {
Verb string
Args []string
Text string
}
func parseFile(reader io.Reader) []logLine {
bufReader := bufio.NewReader(reader)
results := make([]logLine, 0, 512)
for {
line, err := bufReader.ReadString('\n')
if err == io.EOF {
if len(line) < 2 {
break
}
} else if err != nil {
break
}
if len(line) <= 2 {
continue
}
line = strings.Replace(line, "\n", "", 1)
line = strings.Replace(line, "\r", "", 1)
results = append(results, parseLine(line))
}
return results
}
func parseLine(line string) logLine {
textSplit := strings.SplitN(line, " :", 2)
tokens := strings.Split(textSplit[0], " ")
ll := logLine{
Verb: tokens[0],
}
if len(tokens) > 1 {
ll.Args = tokens[1:]
}
if len(textSplit) > 1 {
ll.Text = textSplit[1]
}
return ll
}
func parseFilename(fname string) (date time.Time, channel string, err error) {
date, err = time.ParseInLocation("2006-01-02_150405", fname[:17], time.Local)
if err != nil {
return
}
ms, err := strconv.Atoi(fname[17:20])
if err != nil {
return
}
date = date.Add(time.Duration(ms) * time.Millisecond)
if len(fname) < 23 {
err = errors.New("filename too short")
return
}
channel = fname[21:]
return
}
cmd/rpdata-lb2logimport/main.go
-110
View File
@@ -1,110 +0,0 @@
package main
import (
"fmt"
"log"
"os"
"path"
"strings"
"time"
"git.aiterp.net/rpdata/api/models/posts"
"git.aiterp.net/rpdata/api/internal/store"
"git.aiterp.net/rpdata/api/models/logs"
)
var prefixReplacer = strings.NewReplacer("+", "", "@", "", "!", "", "%", "")
func main() {
err := store.Init()
if err != nil {
log.Fatalln(err)
}
for _, filepath := range os.Args[1:] {
name := strings.Replace(path.Base(filepath), ".txt", "", 1)
file, err := os.Open(filepath)
if err != nil {
log.Println(filepath, err)
continue
}
logLines := parseFile(file)
file.Close()
// Get title and event
title := ""
event := ""
for _, line := range logLines {
if line.Verb == "TITLE" {
title = line.Text
}
if line.Verb == "TAG" {
event = line.Text
}
}
date, channel, err := parseFilename(name)
if err != nil {
log.Fatalln(err)
}
l, err := logs.Add(date, channel, title, event, "", false)
if err != nil {
log.Println(err)
continue
}
_, err = posts.Add(l, time.Now(), "annotation.info", "rpdata-lb2logimport", "This logfile is imported from aitelogs2 and may contain errors or wrong timestamps.")
if err != nil {
log.Println(err)
}
for _, line := range logLines {
if line.Verb != "CHARS" {
continue
}
_, err = posts.Add(l, l.Date, "chars", prefixReplacer.Replace(line.Args[0]), line.Text)
if err != nil {
log.Println(err)
}
}
for _, line := range logLines {
if line.Verb != "SCENE" && line.Verb != "ACTION" && line.Verb != "TEXT" {
continue
}
postTime, err := time.ParseInLocation("2006-01-02 15:04:05", date.Format("2006-01-02")+" "+line.Args[1], time.Local)
diff := postTime.Sub(date)
if err != nil {
log.Println(err)
continue
}
if diff < 0 {
if diff > -time.Second {
postTime = postTime.Add(diff)
} else {
postTime = postTime.Add(time.Hour * 24)
}
}
if line.Args[0][0] == '=' {
line.Verb = "SCENE"
}
_, err = posts.Add(l, postTime, strings.ToLower(line.Verb), prefixReplacer.Replace(line.Args[0]), line.Text)
if err != nil {
log.Println(err)
}
}
l, err = logs.UpdateCharacters(l, nil)
if err != nil {
log.Println(err)
}
fmt.Println(l.ID, "completed")
}
}
cmd/rpdata-wikifileimport/main.go
-77
View File
@@ -1,77 +0,0 @@
package main
import (
"encoding/json"
"fmt"
"log"
"net/http"
"time"
"git.aiterp.net/rpdata/api/internal/config"
"git.aiterp.net/rpdata/api/internal/store"
"git.aiterp.net/rpdata/api/models/files"
)
func main() {
client := http.Client{Timeout: time.Second * 30}
res, err := client.Get(config.Global().Wiki.URL + "?action=query&list=allimages&ailimit=5&aiprop=dimensions|mime|url|user|timestamp&format=json&ailimit=500")
if err != nil {
log.Fatalln(err)
}
if res.StatusCode != 200 {
log.Fatalln(res.Status)
}
data := ResponseData{}
err = json.NewDecoder(res.Body).Decode(&data)
if err != nil {
log.Fatalln(err)
}
store.Init()
for _, info := range data.Query.Allimages {
existing, err := files.FindName("wiki", info.Name, info.User)
if err == nil && existing.ID != "" {
log.Println("Skipping", info.Name, "because it already exists")
continue
}
ts, err := time.Parse(time.RFC3339, info.Timestamp)
if err != nil {
log.Println("Skipping", info.Name, "error:", err)
continue
}
file, err := files.Insert(info.Name, "wiki", info.Mime, info.User, ts, info.Size, info.URL)
if err != nil {
log.Println("Skipping", info.Name, "error:", err)
continue
}
fmt.Println(file.Name, "inserted ( id:", file.ID, ")")
}
}
// The ResponseData from the wiki looks like this
type ResponseData struct {
Batchcomplete string `json:"batchcomplete"`
Query struct {
Allimages []struct {
Descriptionshorturl string `json:"descriptionshorturl"`
Descriptionurl string `json:"descriptionurl"`
Height int `json:"height"`
Mime string `json:"mime"`
Name string `json:"name"`
Ns int `json:"ns"`
Size int64 `json:"size"`
Timestamp string `json:"timestamp"`
Title string `json:"title"`
URL string `json:"url"`
User string `json:"user"`
Width int `json:"width"`
} `json:"allimages"`
} `json:"query"`
}
go.mod
-6
View File
@@ -8,15 +8,10 @@ require (
github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/globalsign/mgo v0.0.0-20180403085842-f76e4f9da92e
github.com/go-ini/ini v1.35.0 // indirect
github.com/go-sql-driver/mysql v1.4.0
github.com/golang/protobuf v1.3.1 // indirect
github.com/google/go-cmp v0.3.0 // indirect
github.com/gorilla/websocket v1.4.0 // indirect
github.com/graph-gophers/dataloader v0.0.0-20180104184831-78139374585c
github.com/h2non/filetype v1.0.8
github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0
github.com/lib/pq v1.1.1 // indirect
github.com/mattn/go-sqlite3 v1.10.0 // indirect
github.com/minio/minio-go v0.0.0-20180409193742-3d2d02921f05
github.com/mitchellh/go-homedir v0.0.0-20161203194507-b8bc1bf76747 // indirect
github.com/prometheus/client_golang v0.9.1
@@ -33,7 +28,6 @@ require (
golang.org/x/net v0.0.0-20190514140710-3ec191127204 // indirect
golang.org/x/sync v0.0.0-20190423024810-112230192c58
golang.org/x/text v0.3.0 // indirect
google.golang.org/appengine v1.1.0 // indirect
gopkg.in/ini.v1 v1.42.0 // indirect
gopkg.in/yaml.v2 v2.2.2
)
go.sum
-12
View File
@@ -21,8 +21,6 @@ github.com/go-ini/ini v1.35.0 h1:D/my3+xOfqZMkJpciRcyqU7XMBUgiZa9qXjZIa8uv2k=
github.com/go-ini/ini v1.35.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk=
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@@ -40,14 +38,10 @@ github.com/gorilla/mux v1.6.1/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z
github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/graph-gophers/dataloader v0.0.0-20180104184831-78139374585c h1:94S+uoVVMpQAEOrqGjCDyUdML4dJDkh6aC4MYmXECg4=
github.com/graph-gophers/dataloader v0.0.0-20180104184831-78139374585c/go.mod h1:jk4jk0c5ZISbKaMe8WsVopGB5/15GvGHMdMdPtwlRp4=
github.com/h2non/filetype v1.0.8 h1:le8gpf+FQA0/DlDABbtisA1KiTS0Xi+YSC/E8yY3Y14=
github.com/h2non/filetype v1.0.8/go.mod h1:isekKqOuhMj+s/7r3rIeTErIRy4Rub5uBWHfvMusLMU=
github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0 h1:5B0uxl2lzNRVkJVg+uGHxWtRt4C0Wjc6kJKo5XYx8xE=
github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
@@ -58,10 +52,6 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4=
github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o=
github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/minio/minio-go v0.0.0-20180409193742-3d2d02921f05 h1:YEyFCqmHcmbcTLqZ7yEAg7VIUat6PuDzGbIctZy888k=
@@ -138,8 +128,6 @@ golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGm
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190515012406-7d7faa4812bd h1:oMEQDWVXVNpceQoVd1JN3CQ7LYJJzs5qWqZIUcxXHHw=
golang.org/x/tools v0.0.0-20190515012406-7d7faa4812bd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
models/files/db.go
-83
View File
@@ -1,83 +0,0 @@
package files
import (
"crypto/rand"
"encoding/binary"
"strconv"
"time"
"git.aiterp.net/rpdata/api/internal/store"
"git.aiterp.net/rpdata/api/models"
"github.com/globalsign/mgo"
)
var collection *mgo.Collection
func find(query interface{}) (models.File, error) {
file := models.File{}
err := collection.Find(query).One(&file)
if err != nil {
return models.File{}, err
}
return file, nil
}
func list(query interface{}) ([]models.File, error) {
list := make([]models.File, 0, 32)
err := collection.Find(query).Sort("-time").All(&list)
if err != nil {
return nil, err
}
return list, nil
}
// makeID makes a random file ID that's 32 characters long
func makeID() string {
result := "F" + strconv.FormatInt(time.Now().UnixNano(), 36)
offset := 0
data := make([]byte, 32)
rand.Read(data)
for len(result) < 32 {
result += strconv.FormatUint(binary.LittleEndian.Uint64(data[offset:]), 36)
offset += 8
if offset >= 32 {
rand.Read(data)
offset = 0
}
}
return result[:32]
}
func init() {
store.HandleInit(func(db *mgo.Database) {
collection = db.C("file.headers")
collection.EnsureIndexKey("author")
collection.EnsureIndexKey("public")
collection.EnsureIndexKey("kind", "name", "author")
collection.EnsureIndexKey("author", "public")
collection.EnsureIndexKey("kind")
})
}
var allowdMimeTypes = map[string]bool{
"": false,
"image/jpeg": true,
"image/png": true,
"image/gif": true,
"image/tiff": true,
"image/tga": true,
"text/plain": true,
"application/json": true,
"application/pdf": false,
"binary/octet-stream": false,
"video/mp4": false,
"audio/mp3": false,
}
models/files/find.go
-16
View File
@@ -1,16 +0,0 @@
package files
import (
"git.aiterp.net/rpdata/api/models"
"github.com/globalsign/mgo/bson"
)
// FindID finds a file by ID
func FindID(id string) (models.File, error) {
return find(bson.M{"_id": id})
}
// FindName finds a file by ID
func FindName(kind, name, author string) (models.File, error) {
return find(bson.M{"name": name, "kind": kind, "author": author})
}
models/files/insert.go
-29
View File
@@ -1,29 +0,0 @@
package files
import (
"time"
"git.aiterp.net/rpdata/api/models"
)
// Insert manually inserts file information into the database. This should never, ever be HTTP API accessible
func Insert(name, kind, mimeType, author string, time time.Time, size int64, url string) (models.File, error) {
file := models.File{
ID: makeID(),
Kind: kind,
Time: time,
Public: false,
Author: author,
Name: name,
MimeType: mimeType,
Size: size,
URL: url,
}
err := collection.Insert(file)
if err != nil {
return models.File{}, err
}
return file, nil
}
models/files/list.go
-36
View File
@@ -1,36 +0,0 @@
package files
import (
"git.aiterp.net/rpdata/api/models"
"github.com/globalsign/mgo/bson"
)
// Filter for files.List
type Filter struct {
Author *string
Public *bool
MimeType []string
}
// List lists files according to the standard lookup. By default it's just the author's own files,
// but if `public` is true it will alos include files made public by other authors. If `mimeTypes` contains
// any, it will limit the results to that. If `author` is empty, it will only list public files
func List(filter *Filter) ([]models.File, error) {
query := bson.M{}
if filter != nil {
if filter.Author != nil {
query["author"] = *filter.Author
}
if filter.Public != nil {
query["public"] = *filter.Public
}
if len(filter.MimeType) > 0 {
query["mimeTypes"] = bson.M{"$in": filter.MimeType}
}
}
return list(query)
}
models/files/upload.go
-52
View File
@@ -1,52 +0,0 @@
package files
import (
"context"
"errors"
"io"
"time"
"git.aiterp.net/rpdata/api/internal/store"
"git.aiterp.net/rpdata/api/models"
)
// Upload adds a file to the space.
func Upload(ctx context.Context, name, mimeType, author string, size int64, input io.Reader) (models.File, error) {
if !allowdMimeTypes[mimeType] {
return models.File{}, errors.New("File type not allowed:" + mimeType)
}
if name == "" {
date := time.Now().UTC().Format("Jan 02 2006 15:04:05 MST")
name = "Unnamed file (" + date + ")"
}
if mimeType == "" {
mimeType = "binary/octet-stream"
}
id := makeID()
path, err := store.UploadFile(ctx, "files", id, mimeType, input, size)
if err != nil {
return models.File{}, err
}
file := models.File{
ID: id,
Kind: "upload",
Time: time.Now(),
Public: false,
Author: author,
Name: name,
MimeType: mimeType,
Size: size,
URL: store.URLFromPath(path),
}
err = collection.Insert(file)
if err != nil {
return models.File{}, err
}
return file, nil
}
models/users/db.go
-14
View File
@@ -1,14 +0,0 @@
package users
import (
"git.aiterp.net/rpdata/api/internal/store"
"github.com/globalsign/mgo"
)
var collection *mgo.Collection
func init() {
store.HandleInit(func(db *mgo.Database) {
collection = db.C("core.users")
})
}
models/users/ensure.go
-34
View File
@@ -1,34 +0,0 @@
package users
import (
"git.aiterp.net/rpdata/api/models"
"github.com/globalsign/mgo"
)
// Ensure finds a user by id, or makes a new one.
func Ensure(id string) (models.User, error) {
user := models.User{}
err := collection.FindId(id).One(&user)
if err == mgo.ErrNotFound {
user = models.User{
ID: id,
Nick: "",
Permissions: []string{
"member",
"log.edit",
"post.edit",
"post.move",
"post.remove",
"file.upload",
},
}
err := collection.Insert(user)
if err != nil {
return models.User{}, err
}
}
return user, err
}
models/users/find.go
-13
View File
@@ -1,13 +0,0 @@
package users
import (
"git.aiterp.net/rpdata/api/models"
)
// Find finds a user by id
func Find(id string) (models.User, error) {
user := models.User{}
err := collection.FindId(id).One(&user)
return user, err
}