From 4979b0210422bb5b06411a59970e652783751e1d Mon Sep 17 00:00:00 2001 From: Gisle Aune Date: Sat, 27 Oct 2018 10:14:58 +0200 Subject: [PATCH] models: Migrated last from model --- model/change/change.go | 105 ---------------- model/change/timeline.go | 118 ------------------ model/file/file.go | 253 --------------------------------------- models/files/db.go | 2 + models/files/find.go | 5 + models/files/insert.go | 29 +++++ models/files/upload.go | 52 ++++++++ 7 files changed, 88 insertions(+), 476 deletions(-) delete mode 100644 model/change/change.go delete mode 100644 model/change/timeline.go delete mode 100644 model/file/file.go create mode 100644 models/files/insert.go create mode 100644 models/files/upload.go diff --git a/model/change/change.go b/model/change/change.go deleted file mode 100644 index a0d80e9..0000000 --- a/model/change/change.go +++ /dev/null @@ -1,105 +0,0 @@ -package change - -import ( - "log" - "sync" - "time" - - "git.aiterp.net/rpdata/api/model/counter" - - "git.aiterp.net/rpdata/api/internal/store" - "github.com/globalsign/mgo" -) - -var collection *mgo.Collection - -// A Change represents a change in any other model -type Change struct { - ID int `bson:"_id" json:"id"` - Time time.Time `bson:"time" json:"time"` - Model string `bson:"model" json:"model"` - Op string `bson:"op" json:"op"` - Author string `bson:"author,omitempty" json:"author,omitempty"` - ObjectID string `bson:"objectId,omitempty" json:"objectId,omitempty"` - Data interface{} `bson:"data,omitempty" json:"data,omitempty"` -} - -// PublicModels lists which models can be listed in bulk by anyone. -var PublicModels = []string{ - "Character", - "Log", - "Post", -} - -var submitMutex sync.Mutex - -// Submit submits a change to the history. -func Submit(model, op, author, objectID string, data interface{}) (Change, error) { - submitMutex.Lock() - defer submitMutex.Unlock() - - index, err := counter.Next("auto_increment", "Change") - if err != nil { - return Change{}, err - } - - change := Change{ - ID: index, - Time: time.Now(), - Model: model, - Op: op, - Author: author, - ObjectID: objectID, - Data: data, - } - - Timeline.push(change) - - err = collection.Insert(&change) - if err != nil { - return Change{}, err - } - - return change, err -} - -// FindID gets a change by ID -func FindID(id int) (Change, error) { - change := Change{} - err := collection.FindId(id).One(&change) - if err != nil { - return Change{}, err - } - - return change, nil -} - -// List gets all changes in ascending order -func List() ([]Change, error) { - changes := make([]Change, 0, 64) - err := collection.Find(nil).Sort("_id").All(&changes) - if err != nil { - return nil, err - } - - return changes, nil -} - -func init() { - store.HandleInit(func(db *mgo.Database) { - collection = db.C("common.history") - - collection.EnsureIndexKey("model") - collection.EnsureIndexKey("author") - collection.EnsureIndexKey("objectId") - - err := collection.EnsureIndex(mgo.Index{ - Name: "expiry", - Key: []string{"time"}, - ExpireAfter: time.Hour * 336, - }) - if err != nil { - log.Fatalln(err) - } - }) -} diff --git a/model/change/timeline.go b/model/change/timeline.go deleted file mode 100644 index e09936b..0000000 --- a/model/change/timeline.go +++ /dev/null @@ -1,118 +0,0 @@ -package change - -import ( - "context" - "errors" - "sync" - - "github.com/globalsign/mgo" -) - -// ErrInvalidIndex is returned when a timeline subscriber tries to get a value -// with a negative id -var ErrInvalidIndex = errors.New("change.timeline: Invalid ID") - -// ErrWaitRequired is returned when a timeline subscriber requests an entry that -// has not yet been pushed -var ErrWaitRequired = errors.New("change.timeline: Waiting required") - -// ErrExpired is returned when a timeline subscriber requests a change that happened -// too long ago for it to be in the database. -var ErrExpired = errors.New("change.timeline: Change has expired") - -type timeline struct { - mutex sync.Mutex - firstIndex int64 - currentID int64 - cache []Change - notifyCh chan struct{} -} - -func (timeline *timeline) Get(ctx context.Context, id int64) (Change, error) { - change, err := timeline.get(id, false) - if err == ErrWaitRequired { - if err := timeline.wait(ctx); err != nil { - return Change{}, err - } - - return timeline.get(id, true) - } else if err == mgo.ErrNotFound { - return Change{}, ErrExpired - } - - return change, nil -} - -func (timeline *timeline) push(change Change) { - timeline.mutex.Lock() - timeline.cache = append(timeline.cache, change) - if len(timeline.cache) > 128 { - copy(timeline.cache, timeline.cache[32:]) - timeline.cache = timeline.cache[:len(timeline.cache)-32] - } - - // Only update the currentID if this is more recent than the existing ones - if int64(change.ID) > timeline.currentID { - timeline.currentID = int64(change.ID) - } - - if timeline.notifyCh != nil { - close(timeline.notifyCh) - timeline.notifyCh = nil - } - timeline.mutex.Unlock() -} - -func (timeline *timeline) get(id int64, hasWaited bool) (Change, error) { - if id < 0 { - return Change{}, ErrInvalidIndex - } - - timeline.mutex.Lock() - if !hasWaited && id > timeline.currentID { - timeline.mutex.Unlock() - return Change{}, ErrWaitRequired - } - - for _, change := range timeline.cache { - if change.ID == int(id) { - timeline.mutex.Unlock() - return change, nil - } - } - timeline.mutex.Unlock() - - return FindID(int(id)) -} - -func (timeline *timeline) wait(ctx context.Context) error { - timeline.mutex.Lock() - if timeline.notifyCh == nil { - timeline.notifyCh = make(chan struct{}) - } - - ch := timeline.notifyCh - timeline.mutex.Unlock() - - select { - case <-ctx.Done(): - return ctx.Err() - case <-ch: - return nil - } -} - -// Timeline has operations for subscribing to changes in real time -var Timeline = &timeline{} - -// InitializeTimeline fills in the timeline with existing entries -func InitializeTimeline() { - changes, err := List() - if err != nil { - return - } - - for _, change := range changes { - Timeline.push(change) - } -} diff --git a/model/file/file.go b/model/file/file.go deleted file mode 100644 index 7c55ada..0000000 --- a/model/file/file.go +++ /dev/null @@ -1,253 +0,0 @@ -package file - -import ( - "context" - "crypto/rand" - "encoding/binary" - "errors" - "io" - "strconv" - "time" - - "git.aiterp.net/rpdata/api/internal/store" - "github.com/globalsign/mgo" - "github.com/globalsign/mgo/bson" -) - -var fileCollection *mgo.Collection - -// A File is a record of a file stored in the Space. -type File struct { - ID string `bson:"_id" json:"id"` - Time time.Time `bson:"time" json:"time"` - Kind string `bson:"kind" json:"kind"` - Public bool `bson:"public" json:"public"` - Name string `bson:"name" json:"name"` - MimeType string `bson:"mimeType" json:"mimeType"` - Size int64 `bson:"size" json:"size"` - Author string `bson:"author" json:"author"` - URL string `bson:"url,omitempty" json:"url,omitempty"` -} - -// Edit edits the file, changing up to both the two mutable properties -func (file *File) Edit(name *string, public *bool) error { - changes := bson.M{} - changedFile := *file - - if name != nil && *name != file.Name { - changes["name"] = *name - changedFile.Name = *name - } - if public != nil && *public != file.Public { - changes["public"] = *public - changedFile.Public = *public - } - - if len(changes) == 0 { - return nil - } - - err := fileCollection.UpdateId(file.ID, bson.M{"$set": changes}) - if err != nil { - return err - } - - *file = changedFile - - return nil -} - -// Delete removes the file information from the database, and deletes the file. -func (file *File) Delete() error { - err := fileCollection.RemoveId(file.ID) - if err != nil { - return err - } - - if file.Kind == "upload" { - err = store.RemoveFile("files", file.ID) - if err != nil { - return err - } - } - - file.URL = "" - - return nil -} - -// Insert manually inserts file information into the database. This should never, ever be API accessible -func Insert(name, kind, mimeType, author string, time time.Time, size int64, url string) (File, error) { - file := File{ - ID: makeFileID(), - Kind: kind, - Time: time, - Public: false, - Author: author, - Name: name, - MimeType: mimeType, - Size: size, - URL: url, - } - - err := fileCollection.Insert(file) - if err != nil { - return File{}, err - } - - return file, nil -} - -// Upload adds a file to the space. -func Upload(ctx context.Context, name, mimeType, author string, size int64, input io.Reader) (File, error) { - if !allowdMimeTypes[mimeType] { - return File{}, errors.New("File type not allowed:" + mimeType) - } - - if name == "" { - date := time.Now().UTC().Format("Jan 02 2006 15:04:05 MST") - name = "Unnamed file (" + date + ")" - } - - if mimeType == "" { - mimeType = "binary/octet-stream" - } - - id := makeFileID() - - path, err := store.UploadFile(ctx, "files", id, mimeType, input, size) - if err != nil { - return File{}, err - } - - file := File{ - ID: id, - Kind: "upload", - Time: time.Now(), - Public: false, - Author: author, - Name: name, - MimeType: mimeType, - Size: size, - URL: store.URLFromPath(path), - } - - err = fileCollection.Insert(file) - if err != nil { - return File{}, err - } - - return file, nil -} - -// FindID finds a file by ID -func FindID(id string) (File, error) { - file := File{} - - err := fileCollection.FindId(id).One(&file) - if err != nil { - return File{}, err - } - - return file, nil -} - -// FindName finds a file by kind, name and author -func FindName(kind string, name string, author string) (File, error) { - query := bson.M{"kind": kind, "name": name, "author": author} - file := File{} - - err := fileCollection.Find(query).One(&file) - if err != nil { - return File{}, err - } - - return file, nil -} - -// List lists files according to the standard lookup. By default it's just the author's own files, -// but if `public` is true it will alos include files made public by other authors. If `mimeTypes` contains -// any, it will limit the results to that. If `author` is empty, it will only list public files -func List(author string, public *bool, mimeTypes []string) ([]File, error) { - query := bson.M{} - - if author != "" { - if public != nil { - if *public == false { - query["author"] = author - } - - query["public"] = *public - } else { - query["author"] = author - } - } else { - if public != nil && *public == false { - return nil, errors.New("You cannot") - } - - query["public"] = true - } - - if len(mimeTypes) > 0 { - query["mimeTypes"] = bson.M{"$in": mimeTypes} - } - - return listFiles(query) -} - -func listFiles(query interface{}) ([]File, error) { - list := make([]File, 0, 32) - - err := fileCollection.Find(query).Sort("-time").All(&list) - if err != nil { - return nil, err - } - - return list, nil -} - -// makeFileID makes a random file ID that's 32 characters long -func makeFileID() string { - result := "F" + strconv.FormatInt(time.Now().UnixNano(), 36) - offset := 0 - data := make([]byte, 32) - - rand.Read(data) - for len(result) < 32 { - result += strconv.FormatUint(binary.LittleEndian.Uint64(data[offset:]), 36) - offset += 8 - - if offset >= 32 { - rand.Read(data) - offset = 0 - } - } - - return result[:32] -} - -func init() { - store.HandleInit(func(db *mgo.Database) { - fileCollection = db.C("file.headers") - - fileCollection.EnsureIndexKey("author") - fileCollection.EnsureIndexKey("public") - fileCollection.EnsureIndexKey("kind", "name", "author") - fileCollection.EnsureIndexKey("author", "public") - fileCollection.EnsureIndexKey("kind") - }) -} - -var allowdMimeTypes = map[string]bool{ - "": false, - "image/jpeg": true, - "image/png": true, - "image/gif": true, - "text/plain": true, - "application/json": true, - "application/pdf": false, - "binary/octet-stream": false, - "video/mp4": false, - "audio/mp3": false, -} diff --git a/models/files/db.go b/models/files/db.go index 7f0af53..a36f392 100644 --- a/models/files/db.go +++ b/models/files/db.go @@ -72,6 +72,8 @@ var allowdMimeTypes = map[string]bool{ "image/jpeg": true, "image/png": true, "image/gif": true, + "image/tiff": true, + "image/tga": true, "text/plain": true, "application/json": true, "application/pdf": false, diff --git a/models/files/find.go b/models/files/find.go index 6fe9d42..de14caf 100644 --- a/models/files/find.go +++ b/models/files/find.go @@ -9,3 +9,8 @@ import ( func FindID(id string) (models.File, error) { return find(bson.M{"_id": id}) } + +// FindName finds a file by ID +func FindName(kind, name, author string) (models.File, error) { + return find(bson.M{"name": name, "kind": kind, "author": author}) +} diff --git a/models/files/insert.go b/models/files/insert.go new file mode 100644 index 0000000..a943912 --- /dev/null +++ b/models/files/insert.go @@ -0,0 +1,29 @@ +package files + +import ( + "time" + + "git.aiterp.net/rpdata/api/models" +) + +// Insert manually inserts file information into the database. This should never, ever be HTTP API accessible +func Insert(name, kind, mimeType, author string, time time.Time, size int64, url string) (models.File, error) { + file := models.File{ + ID: makeID(), + Kind: kind, + Time: time, + Public: false, + Author: author, + Name: name, + MimeType: mimeType, + Size: size, + URL: url, + } + + err := collection.Insert(file) + if err != nil { + return models.File{}, err + } + + return file, nil +} diff --git a/models/files/upload.go b/models/files/upload.go new file mode 100644 index 0000000..e578959 --- /dev/null +++ b/models/files/upload.go @@ -0,0 +1,52 @@ +package files + +import ( + "context" + "errors" + "io" + "time" + + "git.aiterp.net/rpdata/api/internal/store" + "git.aiterp.net/rpdata/api/models" +) + +// Upload adds a file to the space. +func Upload(ctx context.Context, name, mimeType, author string, size int64, input io.Reader) (models.File, error) { + if !allowdMimeTypes[mimeType] { + return models.File{}, errors.New("File type not allowed:" + mimeType) + } + + if name == "" { + date := time.Now().UTC().Format("Jan 02 2006 15:04:05 MST") + name = "Unnamed file (" + date + ")" + } + if mimeType == "" { + mimeType = "binary/octet-stream" + } + + id := makeID() + + path, err := store.UploadFile(ctx, "files", id, mimeType, input, size) + if err != nil { + return models.File{}, err + } + + file := models.File{ + ID: id, + Kind: "upload", + Time: time.Now(), + Public: false, + Author: author, + Name: name, + MimeType: mimeType, + Size: size, + URL: store.URLFromPath(path), + } + + err = collection.Insert(file) + if err != nil { + return models.File{}, err + } + + return file, nil +}