Early work on file stuff.

.idea/dictionaries/gisle.xml

@@ -2,6 +2,7 @@
   <dictionary name="gisle">
     <words>
       <w>logbot</w>
+      <w>minio</w>
       <w>mirc</w>
       <w>rctx</w>
     </words>

database/mongodb/db.go

@@ -20,6 +20,7 @@ type MongoDB struct {
 	tags       repositories.TagRepository
 	logs       *logRepository
 	posts      *postRepository
+	files      *fileRepository
 }
 
 func (m *MongoDB) Changes() repositories.ChangeRepository {
@@ -46,6 +47,10 @@ func (m *MongoDB) Channels() repositories.ChannelRepository {
 	return m.channels
 }
 
+func (m *MongoDB) Files() repositories.FileRepository {
+	return m.files
+}
+
 func (m *MongoDB) Close(ctx context.Context) error {
 	m.session.Close()
 	return nil
@@ -103,6 +108,12 @@ func Init(cfg config.Database) (*MongoDB, error) {
 		return nil, err
 	}
 
+	files, err := newFileRepository(db)
+	if err != nil {
+		session.Close()
+		return nil, err
+	}
+
 	go posts.fixPositions(logs)
 
 	return &MongoDB{
@@ -114,6 +125,7 @@ func Init(cfg config.Database) (*MongoDB, error) {
 		tags:       newTagRepository(db),
 		logs:       logs,
 		posts:      posts,
+		files:      files,
 	}, nil
 }
 

database/mongodb/files.go

@@ -0,0 +1,105 @@
+package mongodb
+
+import (
+	"context"
+	"git.aiterp.net/rpdata/api/internal/generate"
+	"git.aiterp.net/rpdata/api/models"
+	"git.aiterp.net/rpdata/api/repositories"
+	"github.com/globalsign/mgo"
+	"github.com/globalsign/mgo/bson"
+)
+
+type fileRepository struct {
+	files *mgo.Collection
+}
+
+func (r *fileRepository) Find(ctx context.Context, id string) (*models.File, error) {
+	file := new(models.File)
+	err := r.files.FindId(id).One(file)
+	if err != nil {
+		if err == mgo.ErrNotFound {
+			return nil, repositories.ErrNotFound
+		}
+
+		return nil, err
+	}
+
+	return file, nil
+}
+
+func (r *fileRepository) List(ctx context.Context, filter models.FileFilter) ([]*models.File, error) {
+	query := bson.M{}
+	if filter.Author != nil {
+		query["author"] = *filter.Author
+	}
+	if filter.MimeTypes != nil {
+		query["mimeType"] = bson.M{"$in": filter.MimeTypes}
+	}
+	if filter.Public != nil {
+		query["public"] = *filter.Public
+	}
+
+	files := make([]*models.File, 0, 16)
+	err := r.files.Find(query).Sort("-date").All(&files)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return files, nil
+}
+
+func (r *fileRepository) Insert(ctx context.Context, file models.File) (*models.File, error) {
+	file.ID = generate.FileID()
+
+	err := r.files.Insert(&file)
+	if err != nil {
+		return nil, err
+	}
+
+	return &file, nil
+}
+
+func (r *fileRepository) Update(ctx context.Context, file models.File, update models.FileUpdate) (*models.File, error) {
+	updateBson := bson.M{}
+	if update.Public != nil {
+		updateBson["public"] = *update.Public
+		file.Public = *update.Public
+	}
+	if update.Name != nil {
+		updateBson["name"] = *update.Name
+		file.Name = *update.Name
+	}
+
+	err := r.files.UpdateId(file.ID, bson.M{"$set": updateBson})
+	if err != nil {
+		return nil, err
+	}
+
+	return &file, nil
+}
+
+func (r *fileRepository) Delete(ctx context.Context, file models.File) error {
+	return r.files.RemoveId(file.ID)
+}
+
+func newFileRepository(db *mgo.Database) (*fileRepository, error) {
+	files := db.C("file.headers")
+
+	err := files.EnsureIndexKey("author")
+	if err != nil {
+		return nil, err
+	}
+
+	err = files.EnsureIndexKey("mimeType")
+	if err != nil {
+		return nil, err
+	}
+
+	err = files.EnsureIndexKey("public")
+	if err != nil {
+		return nil, err
+	}
+
+	return &fileRepository{files: files}, nil
+}

go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/google/go-cmp v0.3.0 // indirect
 	github.com/gorilla/websocket v1.4.0 // indirect
 	github.com/graph-gophers/dataloader v0.0.0-20180104184831-78139374585c
+	github.com/h2non/filetype v1.0.8
 	github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0
 	github.com/lib/pq v1.1.1 // indirect
 	github.com/mattn/go-sqlite3 v1.10.0 // indirect

go.sum

@@ -42,6 +42,8 @@ github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/graph-gophers/dataloader v0.0.0-20180104184831-78139374585c h1:94S+uoVVMpQAEOrqGjCDyUdML4dJDkh6aC4MYmXECg4=
 github.com/graph-gophers/dataloader v0.0.0-20180104184831-78139374585c/go.mod h1:jk4jk0c5ZISbKaMe8WsVopGB5/15GvGHMdMdPtwlRp4=
+github.com/h2non/filetype v1.0.8 h1:le8gpf+FQA0/DlDABbtisA1KiTS0Xi+YSC/E8yY3Y14=
+github.com/h2non/filetype v1.0.8/go.mod h1:isekKqOuhMj+s/7r3rIeTErIRy4Rub5uBWHfvMusLMU=
 github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0 h1:5B0uxl2lzNRVkJVg+uGHxWtRt4C0Wjc6kJKo5XYx8xE=

internal/auth/permissions.go

@@ -25,6 +25,8 @@ func AllPermissions() map[string]string {
 		"story.remove":     "Can remove non-owned stories",
 		"story.unlisted":   "Can view non-owned unlisted stories",
 		"file.upload":      "Can upload files",
+		"file.list":        "Can list non-owned files",
+		"file.view":        "Can view non-owned files",
 		"file.edit":        "Can edit non-owned files",
 		"file.remove":      "Can remove non-owned files",
 	}

internal/generate/id.go

@@ -52,3 +52,8 @@ func StoryID() string {
 func ChapterID() string {
 	return ID("SC", 24)
 }
+
+// FileID generates a file ID.
+func FileID() string {
+	return ID("F", 16)
+}

models/file.go

@@ -20,3 +20,16 @@ type File struct {
 func (*File) IsChangeObject() {
 	panic("this method is a dummy, and so is its caller")
 }
+
+// A FileFilter is a filter that can be used to filter files.
+type FileFilter struct {
+	Author    *string
+	Public    *bool
+	MimeTypes []string
+}
+
+// A FileUpdate is a set of changes possible to do on file metadata.
+type FileUpdate struct {
+	Public *bool
+	Name   *string
+}

repositories/file.go

@@ -0,0 +1,15 @@
+package repositories
+
+import (
+	"context"
+	"git.aiterp.net/rpdata/api/models"
+)
+
+// FileRepository is an interface for database operations relating to file metadata.
+type FileRepository interface {
+	Find(ctx context.Context, id string) (*models.File, error)
+	List(ctx context.Context, filter models.FileFilter) ([]*models.File, error)
+	Insert(ctx context.Context, file models.File) (*models.File, error)
+	Update(ctx context.Context, file models.File, update models.FileUpdate) (*models.File, error)
+	Delete(ctx context.Context, file models.File) error
+}

services/files.go

@@ -0,0 +1,144 @@
+package services
+
+import (
+	"context"
+	"errors"
+	"git.aiterp.net/rpdata/api/internal/auth"
+	"git.aiterp.net/rpdata/api/models"
+	"git.aiterp.net/rpdata/api/repositories"
+	"github.com/h2non/filetype"
+	"io"
+)
+
+var ErrPrivateNoAuthor = errors.New("cannot search for private files without an author")
+var ErrInvalidName = errors.New("invalid name")
+var ErrInvalidFileSize = errors.New("file is not of a correct size (min: 320B, max: 16MB)")
+var ErrCouldNotReadHead = errors.New("could not read file head")
+var ErrInvalidFileType = errors.New("file type could not be recognized or is not allowed")
+var ErrCouldNotUploadFile = errors.New("could not upload file")
+
+// FileService is a service for files.
+type FileService struct {
+	files repositories.FileRepository
+}
+
+func (s *FileService) Find(ctx context.Context, id string) (*models.File, error) {
+	file, err := s.files.Find(ctx, id)
+	if err != nil {
+		return nil, err
+	}
+
+	if !file.Public {
+		err := auth.CheckPermission(ctx, "view", file)
+		if err != nil {
+			return nil, repositories.ErrNotFound
+		}
+	}
+
+	return file, nil
+}
+
+func (s *FileService) List(ctx context.Context, filter models.FileFilter) ([]*models.File, error) {
+	token := auth.TokenFromContext(ctx)
+
+	if filter.Public != nil {
+		if *filter.Public == false {
+			if filter.Author == nil || *filter.Author == "" {
+				return nil, ErrPrivateNoAuthor
+			}
+
+			if !token.PermittedUser(*filter.Author, "member", "file.list") {
+				return nil, auth.ErrUnauthorized
+			}
+		}
+	}
+
+	return s.files.List(ctx, filter)
+}
+
+func (s *FileService) Upload(ctx context.Context, reader io.Reader, name string, size int64) (*models.File, error) {
+	if name == "" {
+		return nil, ErrInvalidName
+	} else if size < 320 || size > 16777216 {
+		return nil, ErrInvalidFileSize
+	}
+
+	head := make([]byte, 320)
+	n, err := reader.Read(head)
+	if err != nil || n < 320 {
+		return nil, ErrCouldNotReadHead
+	}
+
+	fileType, err := filetype.Match(head)
+	if err != nil || !allowedMimeTypes[fileType.MIME.Value] {
+		return nil, ErrInvalidFileType
+	}
+
+	panic("implement rest of me")
+}
+
+func (s *FileService) Edit(ctx context.Context, id string, name *string, public *bool) (*models.File, error) {
+	file, err := s.files.Find(ctx, id)
+	if err != nil {
+		return nil, err
+	}
+
+	err = auth.CheckPermission(ctx, "edit", file)
+	if err != nil {
+		if !file.Public {
+			return nil, repositories.ErrNotFound
+		}
+
+		return nil, err
+	}
+
+	return s.files.Update(ctx, *file, models.FileUpdate{
+		Name:   name,
+		Public: public,
+	})
+}
+
+// concatReader is a quick and dirty reader for reading the head and the file.
+type concatReader struct {
+	head    []byte
+	headPos int
+	body    io.Reader
+}
+
+func (r *concatReader) Read(p []byte) (n int, err error) {
+	if len(p) == 0 {
+		return 0, io.ErrShortBuffer
+	}
+
+	if r.headPos < len(r.head) {
+		remainder := r.head[r.headPos:]
+		if len(p) < len(remainder) {
+			r.headPos += len(p)
+			copy(p, remainder)
+
+			return len(p), nil
+		}
+
+		r.headPos = len(r.head)
+		copy(p, remainder)
+
+		return len(p), nil
+	}
+
+	return r.body.Read(p)
+}
+
+var allowedMimeTypes = map[string]bool{
+	"":                    false,
+	"image/jpeg":          true,
+	"image/png":           true,
+	"image/gif":           true,
+	"image/tiff":          true,
+	"image/tga":           true,
+	"text/plain":          true,
+	"application/json":    true,
+	"application/pdf":     false,
+	"binary/octet-stream": false,
+	"video/mp4":           false,
+	"audio/mp3":           false,
+}

space/space.go

@@ -0,0 +1,93 @@
+package space
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"git.aiterp.net/rpdata/api/internal/config"
+	"io"
+
+	minio "github.com/minio/minio-go"
+)
+
+type Client struct {
+	bucket    string
+	urlRoot   string
+	spaceRoot string
+	maxSize   int64
+
+	s3 *minio.Client
+}
+
+func (client *Client) MaxSize() int64 {
+	return client.maxSize
+}
+
+// UploadFile uploads the file to the space. This does not do any checks on it, so the endpoints should
+// ensure that's all okay.
+func (client *Client) UploadFile(ctx context.Context, folder string, name string, mimeType string, reader io.Reader, size int64) (string, error) {
+	path := folder + "/" + name
+
+	if size > client.maxSize {
+		return "", errors.New("file is too big")
+	}
+
+	_, err := client.s3.PutObjectWithContext(ctx, client.bucket, client.spaceRoot+"/"+path, reader, size, minio.PutObjectOptions{
+		ContentType: mimeType,
+		UserMetadata: map[string]string{
+			"x-amz-acl": "public-read",
+		},
+	})
+	if err != nil {
+		return "", err
+	}
+
+	_, err = client.s3.StatObject(client.bucket, client.spaceRoot+"/"+path, minio.StatObjectOptions{})
+	if err != nil {
+		return "", err
+	}
+
+	return path, nil
+}
+
+// RemoveFile removes a file from the space
+func (client *Client) RemoveFile(folder string, name string) error {
+	path := folder + "/" + name
+
+	return client.s3.RemoveObject(client.bucket, client.spaceRoot+"/"+path)
+}
+
+// DownloadFile opens a file for download, using the same path format as the UploadFile function. Remember to Close it!
+// The returned stream also has `ReaderAt` and `Seeker`, but those are impl. details.
+func (client *Client) DownloadFile(ctx context.Context, path string) (io.ReadCloser, error) {
+	return client.s3.GetObjectWithContext(ctx, client.bucket, client.spaceRoot+"/"+path, minio.GetObjectOptions{})
+}
+
+// URLFromPath gets the URL from the path returned by UploadFile
+func (client *Client) URLFromPath(path string) string {
+	return client.urlRoot + path
+}
+
+// Connect connects to a S3 space.
+func Connect(cfg config.Space) (*Client, error) {
+	client, err := minio.New(cfg.Host, cfg.AccessKey, cfg.SecretKey, true)
+	if err != nil {
+		return nil, err
+	}
+
+	exists, err := client.BucketExists(cfg.Bucket)
+	if err != nil {
+		return nil, err
+	}
+	if !exists {
+		return nil, errors.New("bucket not found")
+	}
+
+	return &Client{
+		bucket:    cfg.Bucket,
+		urlRoot:   fmt.Sprintf("https://%s.%s/%s/", cfg.Bucket, cfg.Host, cfg.Root),
+		spaceRoot: cfg.Root,
+		maxSize:   cfg.MaxSize,
+		s3:        client,
+	}, nil
+}