Browse Source

Early work on file stuff.

thegreatrefactor
Gisle Aune 5 years ago
parent
commit
5bcc7479ff
  1. 1
      .idea/dictionaries/gisle.xml
  2. 12
      database/mongodb/db.go
  3. 105
      database/mongodb/files.go
  4. 1
      go.mod
  5. 2
      go.sum
  6. 2
      internal/auth/permissions.go
  7. 5
      internal/generate/id.go
  8. 15
      models/file.go
  9. 15
      repositories/file.go
  10. 144
      services/files.go
  11. 93
      space/space.go

1
.idea/dictionaries/gisle.xml

@ -2,6 +2,7 @@
<dictionary name="gisle"> <dictionary name="gisle">
<words> <words>
<w>logbot</w> <w>logbot</w>
<w>minio</w>
<w>mirc</w> <w>mirc</w>
<w>rctx</w> <w>rctx</w>
</words> </words>

12
database/mongodb/db.go

@ -20,6 +20,7 @@ type MongoDB struct {
tags repositories.TagRepository tags repositories.TagRepository
logs *logRepository logs *logRepository
posts *postRepository posts *postRepository
files *fileRepository
} }
func (m *MongoDB) Changes() repositories.ChangeRepository { func (m *MongoDB) Changes() repositories.ChangeRepository {
@ -46,6 +47,10 @@ func (m *MongoDB) Channels() repositories.ChannelRepository {
return m.channels return m.channels
} }
func (m *MongoDB) Files() repositories.FileRepository {
return m.files
}
func (m *MongoDB) Close(ctx context.Context) error { func (m *MongoDB) Close(ctx context.Context) error {
m.session.Close() m.session.Close()
return nil return nil
@ -103,6 +108,12 @@ func Init(cfg config.Database) (*MongoDB, error) {
return nil, err return nil, err
} }
files, err := newFileRepository(db)
if err != nil {
session.Close()
return nil, err
}
go posts.fixPositions(logs) go posts.fixPositions(logs)
return &MongoDB{ return &MongoDB{
@ -114,6 +125,7 @@ func Init(cfg config.Database) (*MongoDB, error) {
tags: newTagRepository(db), tags: newTagRepository(db),
logs: logs, logs: logs,
posts: posts, posts: posts,
files: files,
}, nil }, nil
} }

105
database/mongodb/files.go

@ -0,0 +1,105 @@
package mongodb
import (
"context"
"git.aiterp.net/rpdata/api/internal/generate"
"git.aiterp.net/rpdata/api/models"
"git.aiterp.net/rpdata/api/repositories"
"github.com/globalsign/mgo"
"github.com/globalsign/mgo/bson"
)
type fileRepository struct {
files *mgo.Collection
}
func (r *fileRepository) Find(ctx context.Context, id string) (*models.File, error) {
file := new(models.File)
err := r.files.FindId(id).One(file)
if err != nil {
if err == mgo.ErrNotFound {
return nil, repositories.ErrNotFound
}
return nil, err
}
return file, nil
}
func (r *fileRepository) List(ctx context.Context, filter models.FileFilter) ([]*models.File, error) {
query := bson.M{}
if filter.Author != nil {
query["author"] = *filter.Author
}
if filter.MimeTypes != nil {
query["mimeType"] = bson.M{"$in": filter.MimeTypes}
}
if filter.Public != nil {
query["public"] = *filter.Public
}
files := make([]*models.File, 0, 16)
err := r.files.Find(query).Sort("-date").All(&files)
if err != nil {
return nil, err
}
return files, nil
}
func (r *fileRepository) Insert(ctx context.Context, file models.File) (*models.File, error) {
file.ID = generate.FileID()
err := r.files.Insert(&file)
if err != nil {
return nil, err
}
return &file, nil
}
func (r *fileRepository) Update(ctx context.Context, file models.File, update models.FileUpdate) (*models.File, error) {
updateBson := bson.M{}
if update.Public != nil {
updateBson["public"] = *update.Public
file.Public = *update.Public
}
if update.Name != nil {
updateBson["name"] = *update.Name
file.Name = *update.Name
}
err := r.files.UpdateId(file.ID, bson.M{"$set": updateBson})
if err != nil {
return nil, err
}
return &file, nil
}
func (r *fileRepository) Delete(ctx context.Context, file models.File) error {
return r.files.RemoveId(file.ID)
}
func newFileRepository(db *mgo.Database) (*fileRepository, error) {
files := db.C("file.headers")
err := files.EnsureIndexKey("author")
if err != nil {
return nil, err
}
err = files.EnsureIndexKey("mimeType")
if err != nil {
return nil, err
}
err = files.EnsureIndexKey("public")
if err != nil {
return nil, err
}
return &fileRepository{files: files}, nil
}

1
go.mod

@ -13,6 +13,7 @@ require (
github.com/google/go-cmp v0.3.0 // indirect github.com/google/go-cmp v0.3.0 // indirect
github.com/gorilla/websocket v1.4.0 // indirect github.com/gorilla/websocket v1.4.0 // indirect
github.com/graph-gophers/dataloader v0.0.0-20180104184831-78139374585c github.com/graph-gophers/dataloader v0.0.0-20180104184831-78139374585c
github.com/h2non/filetype v1.0.8
github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0 github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0
github.com/lib/pq v1.1.1 // indirect github.com/lib/pq v1.1.1 // indirect
github.com/mattn/go-sqlite3 v1.10.0 // indirect github.com/mattn/go-sqlite3 v1.10.0 // indirect

2
go.sum

@ -42,6 +42,8 @@ github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/graph-gophers/dataloader v0.0.0-20180104184831-78139374585c h1:94S+uoVVMpQAEOrqGjCDyUdML4dJDkh6aC4MYmXECg4= github.com/graph-gophers/dataloader v0.0.0-20180104184831-78139374585c h1:94S+uoVVMpQAEOrqGjCDyUdML4dJDkh6aC4MYmXECg4=
github.com/graph-gophers/dataloader v0.0.0-20180104184831-78139374585c/go.mod h1:jk4jk0c5ZISbKaMe8WsVopGB5/15GvGHMdMdPtwlRp4= github.com/graph-gophers/dataloader v0.0.0-20180104184831-78139374585c/go.mod h1:jk4jk0c5ZISbKaMe8WsVopGB5/15GvGHMdMdPtwlRp4=
github.com/h2non/filetype v1.0.8 h1:le8gpf+FQA0/DlDABbtisA1KiTS0Xi+YSC/E8yY3Y14=
github.com/h2non/filetype v1.0.8/go.mod h1:isekKqOuhMj+s/7r3rIeTErIRy4Rub5uBWHfvMusLMU=
github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0 h1:5B0uxl2lzNRVkJVg+uGHxWtRt4C0Wjc6kJKo5XYx8xE= github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0 h1:5B0uxl2lzNRVkJVg+uGHxWtRt4C0Wjc6kJKo5XYx8xE=

2
internal/auth/permissions.go

@ -25,6 +25,8 @@ func AllPermissions() map[string]string {
"story.remove": "Can remove non-owned stories", "story.remove": "Can remove non-owned stories",
"story.unlisted": "Can view non-owned unlisted stories", "story.unlisted": "Can view non-owned unlisted stories",
"file.upload": "Can upload files", "file.upload": "Can upload files",
"file.list": "Can list non-owned files",
"file.view": "Can view non-owned files",
"file.edit": "Can edit non-owned files", "file.edit": "Can edit non-owned files",
"file.remove": "Can remove non-owned files", "file.remove": "Can remove non-owned files",
} }

5
internal/generate/id.go

@ -52,3 +52,8 @@ func StoryID() string {
func ChapterID() string { func ChapterID() string {
return ID("SC", 24) return ID("SC", 24)
} }
// FileID generates a file ID.
func FileID() string {
return ID("F", 16)
}

15
models/file.go

@ -19,4 +19,17 @@ type File struct {
// ChangeObject in GQL. // ChangeObject in GQL.
func (*File) IsChangeObject() { func (*File) IsChangeObject() {
panic("this method is a dummy, and so is its caller") panic("this method is a dummy, and so is its caller")
}
}
// A FileFilter is a filter that can be used to filter files.
type FileFilter struct {
Author *string
Public *bool
MimeTypes []string
}
// A FileUpdate is a set of changes possible to do on file metadata.
type FileUpdate struct {
Public *bool
Name *string
}

15
repositories/file.go

@ -0,0 +1,15 @@
package repositories
import (
"context"
"git.aiterp.net/rpdata/api/models"
)
// FileRepository is an interface for database operations relating to file metadata.
type FileRepository interface {
Find(ctx context.Context, id string) (*models.File, error)
List(ctx context.Context, filter models.FileFilter) ([]*models.File, error)
Insert(ctx context.Context, file models.File) (*models.File, error)
Update(ctx context.Context, file models.File, update models.FileUpdate) (*models.File, error)
Delete(ctx context.Context, file models.File) error
}

144
services/files.go

@ -0,0 +1,144 @@
package services
import (
"context"
"errors"
"git.aiterp.net/rpdata/api/internal/auth"
"git.aiterp.net/rpdata/api/models"
"git.aiterp.net/rpdata/api/repositories"
"github.com/h2non/filetype"
"io"
)
var ErrPrivateNoAuthor = errors.New("cannot search for private files without an author")
var ErrInvalidName = errors.New("invalid name")
var ErrInvalidFileSize = errors.New("file is not of a correct size (min: 320B, max: 16MB)")
var ErrCouldNotReadHead = errors.New("could not read file head")
var ErrInvalidFileType = errors.New("file type could not be recognized or is not allowed")
var ErrCouldNotUploadFile = errors.New("could not upload file")
// FileService is a service for files.
type FileService struct {
files repositories.FileRepository
}
func (s *FileService) Find(ctx context.Context, id string) (*models.File, error) {
file, err := s.files.Find(ctx, id)
if err != nil {
return nil, err
}
if !file.Public {
err := auth.CheckPermission(ctx, "view", file)
if err != nil {
return nil, repositories.ErrNotFound
}
}
return file, nil
}
func (s *FileService) List(ctx context.Context, filter models.FileFilter) ([]*models.File, error) {
token := auth.TokenFromContext(ctx)
if filter.Public != nil {
if *filter.Public == false {
if filter.Author == nil || *filter.Author == "" {
return nil, ErrPrivateNoAuthor
}
if !token.PermittedUser(*filter.Author, "member", "file.list") {
return nil, auth.ErrUnauthorized
}
}
}
return s.files.List(ctx, filter)
}
func (s *FileService) Upload(ctx context.Context, reader io.Reader, name string, size int64) (*models.File, error) {
if name == "" {
return nil, ErrInvalidName
} else if size < 320 || size > 16777216 {
return nil, ErrInvalidFileSize
}
head := make([]byte, 320)
n, err := reader.Read(head)
if err != nil || n < 320 {
return nil, ErrCouldNotReadHead
}
fileType, err := filetype.Match(head)
if err != nil || !allowedMimeTypes[fileType.MIME.Value] {
return nil, ErrInvalidFileType
}
panic("implement rest of me")
}
func (s *FileService) Edit(ctx context.Context, id string, name *string, public *bool) (*models.File, error) {
file, err := s.files.Find(ctx, id)
if err != nil {
return nil, err
}
err = auth.CheckPermission(ctx, "edit", file)
if err != nil {
if !file.Public {
return nil, repositories.ErrNotFound
}
return nil, err
}
return s.files.Update(ctx, *file, models.FileUpdate{
Name: name,
Public: public,
})
}
// concatReader is a quick and dirty reader for reading the head and the file.
type concatReader struct {
head []byte
headPos int
body io.Reader
}
func (r *concatReader) Read(p []byte) (n int, err error) {
if len(p) == 0 {
return 0, io.ErrShortBuffer
}
if r.headPos < len(r.head) {
remainder := r.head[r.headPos:]
if len(p) < len(remainder) {
r.headPos += len(p)
copy(p, remainder)
return len(p), nil
}
r.headPos = len(r.head)
copy(p, remainder)
return len(p), nil
}
return r.body.Read(p)
}
var allowedMimeTypes = map[string]bool{
"": false,
"image/jpeg": true,
"image/png": true,
"image/gif": true,
"image/tiff": true,
"image/tga": true,
"text/plain": true,
"application/json": true,
"application/pdf": false,
"binary/octet-stream": false,
"video/mp4": false,
"audio/mp3": false,
}

93
space/space.go

@ -0,0 +1,93 @@
package space
import (
"context"
"errors"
"fmt"
"git.aiterp.net/rpdata/api/internal/config"
"io"
minio "github.com/minio/minio-go"
)
type Client struct {
bucket string
urlRoot string
spaceRoot string
maxSize int64
s3 *minio.Client
}
func (client *Client) MaxSize() int64 {
return client.maxSize
}
// UploadFile uploads the file to the space. This does not do any checks on it, so the endpoints should
// ensure that's all okay.
func (client *Client) UploadFile(ctx context.Context, folder string, name string, mimeType string, reader io.Reader, size int64) (string, error) {
path := folder + "/" + name
if size > client.maxSize {
return "", errors.New("file is too big")
}
_, err := client.s3.PutObjectWithContext(ctx, client.bucket, client.spaceRoot+"/"+path, reader, size, minio.PutObjectOptions{
ContentType: mimeType,
UserMetadata: map[string]string{
"x-amz-acl": "public-read",
},
})
if err != nil {
return "", err
}
_, err = client.s3.StatObject(client.bucket, client.spaceRoot+"/"+path, minio.StatObjectOptions{})
if err != nil {
return "", err
}
return path, nil
}
// RemoveFile removes a file from the space
func (client *Client) RemoveFile(folder string, name string) error {
path := folder + "/" + name
return client.s3.RemoveObject(client.bucket, client.spaceRoot+"/"+path)
}
// DownloadFile opens a file for download, using the same path format as the UploadFile function. Remember to Close it!
// The returned stream also has `ReaderAt` and `Seeker`, but those are impl. details.
func (client *Client) DownloadFile(ctx context.Context, path string) (io.ReadCloser, error) {
return client.s3.GetObjectWithContext(ctx, client.bucket, client.spaceRoot+"/"+path, minio.GetObjectOptions{})
}
// URLFromPath gets the URL from the path returned by UploadFile
func (client *Client) URLFromPath(path string) string {
return client.urlRoot + path
}
// Connect connects to a S3 space.
func Connect(cfg config.Space) (*Client, error) {
client, err := minio.New(cfg.Host, cfg.AccessKey, cfg.SecretKey, true)
if err != nil {
return nil, err
}
exists, err := client.BucketExists(cfg.Bucket)
if err != nil {
return nil, err
}
if !exists {
return nil, errors.New("bucket not found")
}
return &Client{
bucket: cfg.Bucket,
urlRoot: fmt.Sprintf("https://%s.%s/%s/", cfg.Bucket, cfg.Host, cfg.Root),
spaceRoot: cfg.Root,
maxSize: cfg.MaxSize,
s3: client,
}, nil
}
Loading…
Cancel
Save