diff --git a/.idea/dictionaries/gisle.xml b/.idea/dictionaries/gisle.xml index 6849365..78a307e 100644 --- a/.idea/dictionaries/gisle.xml +++ b/.idea/dictionaries/gisle.xml @@ -2,6 +2,7 @@ logbot + minio mirc rctx diff --git a/database/mongodb/db.go b/database/mongodb/db.go index 39ae376..e8b6bbe 100644 --- a/database/mongodb/db.go +++ b/database/mongodb/db.go @@ -20,6 +20,7 @@ type MongoDB struct { tags repositories.TagRepository logs *logRepository posts *postRepository + files *fileRepository } func (m *MongoDB) Changes() repositories.ChangeRepository { @@ -46,6 +47,10 @@ func (m *MongoDB) Channels() repositories.ChannelRepository { return m.channels } +func (m *MongoDB) Files() repositories.FileRepository { + return m.files +} + func (m *MongoDB) Close(ctx context.Context) error { m.session.Close() return nil @@ -103,6 +108,12 @@ func Init(cfg config.Database) (*MongoDB, error) { return nil, err } + files, err := newFileRepository(db) + if err != nil { + session.Close() + return nil, err + } + go posts.fixPositions(logs) return &MongoDB{ @@ -114,6 +125,7 @@ func Init(cfg config.Database) (*MongoDB, error) { tags: newTagRepository(db), logs: logs, posts: posts, + files: files, }, nil } diff --git a/database/mongodb/files.go b/database/mongodb/files.go new file mode 100644 index 0000000..51816b0 --- /dev/null +++ b/database/mongodb/files.go @@ -0,0 +1,105 @@ +package mongodb + +import ( + "context" + "git.aiterp.net/rpdata/api/internal/generate" + "git.aiterp.net/rpdata/api/models" + "git.aiterp.net/rpdata/api/repositories" + "github.com/globalsign/mgo" + "github.com/globalsign/mgo/bson" +) + +type fileRepository struct { + files *mgo.Collection +} + +func (r *fileRepository) Find(ctx context.Context, id string) (*models.File, error) { + file := new(models.File) + err := r.files.FindId(id).One(file) + if err != nil { + if err == mgo.ErrNotFound { + return nil, repositories.ErrNotFound + } + + return nil, err + } + + return file, nil +} + +func (r *fileRepository) List(ctx context.Context, filter models.FileFilter) ([]*models.File, error) { + query := bson.M{} + if filter.Author != nil { + query["author"] = *filter.Author + } + if filter.MimeTypes != nil { + query["mimeType"] = bson.M{"$in": filter.MimeTypes} + } + if filter.Public != nil { + query["public"] = *filter.Public + } + + files := make([]*models.File, 0, 16) + err := r.files.Find(query).Sort("-date").All(&files) + + if err != nil { + return nil, err + } + + return files, nil +} + +func (r *fileRepository) Insert(ctx context.Context, file models.File) (*models.File, error) { + file.ID = generate.FileID() + + err := r.files.Insert(&file) + if err != nil { + return nil, err + } + + return &file, nil +} + +func (r *fileRepository) Update(ctx context.Context, file models.File, update models.FileUpdate) (*models.File, error) { + updateBson := bson.M{} + if update.Public != nil { + updateBson["public"] = *update.Public + file.Public = *update.Public + } + if update.Name != nil { + updateBson["name"] = *update.Name + file.Name = *update.Name + } + + err := r.files.UpdateId(file.ID, bson.M{"$set": updateBson}) + if err != nil { + return nil, err + } + + return &file, nil +} + +func (r *fileRepository) Delete(ctx context.Context, file models.File) error { + return r.files.RemoveId(file.ID) +} + +func newFileRepository(db *mgo.Database) (*fileRepository, error) { + files := db.C("file.headers") + + err := files.EnsureIndexKey("author") + if err != nil { + return nil, err + } + + err = files.EnsureIndexKey("mimeType") + if err != nil { + return nil, err + } + + err = files.EnsureIndexKey("public") + if err != nil { + return nil, err + } + + return &fileRepository{files: files}, nil +} diff --git a/go.mod b/go.mod index 06168ce..e7a9205 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/google/go-cmp v0.3.0 // indirect github.com/gorilla/websocket v1.4.0 // indirect github.com/graph-gophers/dataloader v0.0.0-20180104184831-78139374585c + github.com/h2non/filetype v1.0.8 github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0 github.com/lib/pq v1.1.1 // indirect github.com/mattn/go-sqlite3 v1.10.0 // indirect diff --git a/go.sum b/go.sum index 54a1259..b67cadb 100644 --- a/go.sum +++ b/go.sum @@ -42,6 +42,8 @@ github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/graph-gophers/dataloader v0.0.0-20180104184831-78139374585c h1:94S+uoVVMpQAEOrqGjCDyUdML4dJDkh6aC4MYmXECg4= github.com/graph-gophers/dataloader v0.0.0-20180104184831-78139374585c/go.mod h1:jk4jk0c5ZISbKaMe8WsVopGB5/15GvGHMdMdPtwlRp4= +github.com/h2non/filetype v1.0.8 h1:le8gpf+FQA0/DlDABbtisA1KiTS0Xi+YSC/E8yY3Y14= +github.com/h2non/filetype v1.0.8/go.mod h1:isekKqOuhMj+s/7r3rIeTErIRy4Rub5uBWHfvMusLMU= github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0 h1:5B0uxl2lzNRVkJVg+uGHxWtRt4C0Wjc6kJKo5XYx8xE= diff --git a/internal/auth/permissions.go b/internal/auth/permissions.go index 216ace6..48d8445 100644 --- a/internal/auth/permissions.go +++ b/internal/auth/permissions.go @@ -25,6 +25,8 @@ func AllPermissions() map[string]string { "story.remove": "Can remove non-owned stories", "story.unlisted": "Can view non-owned unlisted stories", "file.upload": "Can upload files", + "file.list": "Can list non-owned files", + "file.view": "Can view non-owned files", "file.edit": "Can edit non-owned files", "file.remove": "Can remove non-owned files", } diff --git a/internal/generate/id.go b/internal/generate/id.go index 95c2d8b..58bc30e 100644 --- a/internal/generate/id.go +++ b/internal/generate/id.go @@ -52,3 +52,8 @@ func StoryID() string { func ChapterID() string { return ID("SC", 24) } + +// FileID generates a file ID. +func FileID() string { + return ID("F", 16) +} diff --git a/models/file.go b/models/file.go index 4da021b..572ef24 100644 --- a/models/file.go +++ b/models/file.go @@ -19,4 +19,17 @@ type File struct { // ChangeObject in GQL. func (*File) IsChangeObject() { panic("this method is a dummy, and so is its caller") -} \ No newline at end of file +} + +// A FileFilter is a filter that can be used to filter files. +type FileFilter struct { + Author *string + Public *bool + MimeTypes []string +} + +// A FileUpdate is a set of changes possible to do on file metadata. +type FileUpdate struct { + Public *bool + Name *string +} diff --git a/repositories/file.go b/repositories/file.go new file mode 100644 index 0000000..160cfa1 --- /dev/null +++ b/repositories/file.go @@ -0,0 +1,15 @@ +package repositories + +import ( + "context" + "git.aiterp.net/rpdata/api/models" +) + +// FileRepository is an interface for database operations relating to file metadata. +type FileRepository interface { + Find(ctx context.Context, id string) (*models.File, error) + List(ctx context.Context, filter models.FileFilter) ([]*models.File, error) + Insert(ctx context.Context, file models.File) (*models.File, error) + Update(ctx context.Context, file models.File, update models.FileUpdate) (*models.File, error) + Delete(ctx context.Context, file models.File) error +} diff --git a/services/files.go b/services/files.go new file mode 100644 index 0000000..c6eb450 --- /dev/null +++ b/services/files.go @@ -0,0 +1,144 @@ +package services + +import ( + "context" + "errors" + "git.aiterp.net/rpdata/api/internal/auth" + "git.aiterp.net/rpdata/api/models" + "git.aiterp.net/rpdata/api/repositories" + "github.com/h2non/filetype" + "io" +) + +var ErrPrivateNoAuthor = errors.New("cannot search for private files without an author") +var ErrInvalidName = errors.New("invalid name") +var ErrInvalidFileSize = errors.New("file is not of a correct size (min: 320B, max: 16MB)") +var ErrCouldNotReadHead = errors.New("could not read file head") +var ErrInvalidFileType = errors.New("file type could not be recognized or is not allowed") +var ErrCouldNotUploadFile = errors.New("could not upload file") + +// FileService is a service for files. +type FileService struct { + files repositories.FileRepository +} + +func (s *FileService) Find(ctx context.Context, id string) (*models.File, error) { + file, err := s.files.Find(ctx, id) + if err != nil { + return nil, err + } + + if !file.Public { + err := auth.CheckPermission(ctx, "view", file) + if err != nil { + return nil, repositories.ErrNotFound + } + } + + return file, nil +} + +func (s *FileService) List(ctx context.Context, filter models.FileFilter) ([]*models.File, error) { + token := auth.TokenFromContext(ctx) + + if filter.Public != nil { + if *filter.Public == false { + if filter.Author == nil || *filter.Author == "" { + return nil, ErrPrivateNoAuthor + } + + if !token.PermittedUser(*filter.Author, "member", "file.list") { + return nil, auth.ErrUnauthorized + } + } + } + + return s.files.List(ctx, filter) +} + +func (s *FileService) Upload(ctx context.Context, reader io.Reader, name string, size int64) (*models.File, error) { + if name == "" { + return nil, ErrInvalidName + } else if size < 320 || size > 16777216 { + return nil, ErrInvalidFileSize + } + + head := make([]byte, 320) + n, err := reader.Read(head) + if err != nil || n < 320 { + return nil, ErrCouldNotReadHead + } + + fileType, err := filetype.Match(head) + if err != nil || !allowedMimeTypes[fileType.MIME.Value] { + return nil, ErrInvalidFileType + } + + panic("implement rest of me") +} + +func (s *FileService) Edit(ctx context.Context, id string, name *string, public *bool) (*models.File, error) { + file, err := s.files.Find(ctx, id) + if err != nil { + return nil, err + } + + err = auth.CheckPermission(ctx, "edit", file) + if err != nil { + if !file.Public { + return nil, repositories.ErrNotFound + } + + return nil, err + } + + return s.files.Update(ctx, *file, models.FileUpdate{ + Name: name, + Public: public, + }) +} + +// concatReader is a quick and dirty reader for reading the head and the file. +type concatReader struct { + head []byte + headPos int + body io.Reader +} + +func (r *concatReader) Read(p []byte) (n int, err error) { + if len(p) == 0 { + return 0, io.ErrShortBuffer + } + + if r.headPos < len(r.head) { + remainder := r.head[r.headPos:] + if len(p) < len(remainder) { + r.headPos += len(p) + copy(p, remainder) + + return len(p), nil + } + + r.headPos = len(r.head) + copy(p, remainder) + + return len(p), nil + } + + return r.body.Read(p) +} + +var allowedMimeTypes = map[string]bool{ + "": false, + "image/jpeg": true, + "image/png": true, + "image/gif": true, + "image/tiff": true, + "image/tga": true, + "text/plain": true, + "application/json": true, + "application/pdf": false, + "binary/octet-stream": false, + "video/mp4": false, + "audio/mp3": false, +} diff --git a/space/space.go b/space/space.go new file mode 100644 index 0000000..9eff8d3 --- /dev/null +++ b/space/space.go @@ -0,0 +1,93 @@ +package space + +import ( + "context" + "errors" + "fmt" + "git.aiterp.net/rpdata/api/internal/config" + "io" + + minio "github.com/minio/minio-go" +) + +type Client struct { + bucket string + urlRoot string + spaceRoot string + maxSize int64 + + s3 *minio.Client +} + +func (client *Client) MaxSize() int64 { + return client.maxSize +} + +// UploadFile uploads the file to the space. This does not do any checks on it, so the endpoints should +// ensure that's all okay. +func (client *Client) UploadFile(ctx context.Context, folder string, name string, mimeType string, reader io.Reader, size int64) (string, error) { + path := folder + "/" + name + + if size > client.maxSize { + return "", errors.New("file is too big") + } + + _, err := client.s3.PutObjectWithContext(ctx, client.bucket, client.spaceRoot+"/"+path, reader, size, minio.PutObjectOptions{ + ContentType: mimeType, + UserMetadata: map[string]string{ + "x-amz-acl": "public-read", + }, + }) + if err != nil { + return "", err + } + + _, err = client.s3.StatObject(client.bucket, client.spaceRoot+"/"+path, minio.StatObjectOptions{}) + if err != nil { + return "", err + } + + return path, nil +} + +// RemoveFile removes a file from the space +func (client *Client) RemoveFile(folder string, name string) error { + path := folder + "/" + name + + return client.s3.RemoveObject(client.bucket, client.spaceRoot+"/"+path) +} + +// DownloadFile opens a file for download, using the same path format as the UploadFile function. Remember to Close it! +// The returned stream also has `ReaderAt` and `Seeker`, but those are impl. details. +func (client *Client) DownloadFile(ctx context.Context, path string) (io.ReadCloser, error) { + return client.s3.GetObjectWithContext(ctx, client.bucket, client.spaceRoot+"/"+path, minio.GetObjectOptions{}) +} + +// URLFromPath gets the URL from the path returned by UploadFile +func (client *Client) URLFromPath(path string) string { + return client.urlRoot + path +} + +// Connect connects to a S3 space. +func Connect(cfg config.Space) (*Client, error) { + client, err := minio.New(cfg.Host, cfg.AccessKey, cfg.SecretKey, true) + if err != nil { + return nil, err + } + + exists, err := client.BucketExists(cfg.Bucket) + if err != nil { + return nil, err + } + if !exists { + return nil, errors.New("bucket not found") + } + + return &Client{ + bucket: cfg.Bucket, + urlRoot: fmt.Sprintf("https://%s.%s/%s/", cfg.Bucket, cfg.Host, cfg.Root), + spaceRoot: cfg.Root, + maxSize: cfg.MaxSize, + s3: client, + }, nil +}