Browse Source

Moved channels to new system, fixed change insert not working.

thegreatrefactor
Gisle Aune 5 years ago
parent
commit
acb6b637a5
  1. 2
      database/mongodb/changes.go
  2. 29
      database/mongodb/channels.go
  3. 7
      database/mongodb/db.go
  4. 3
      graph2/complexity.go
  5. 2
      graph2/gqlgen.yml
  6. 79
      graph2/resolvers/channel.go
  7. 12
      graph2/types/log.go
  8. 16
      models/change.go
  9. 9
      models/channel.go
  10. 5
      services/changes.go
  11. 137
      services/channels.go
  12. 1
      services/characters.go
  13. 224
      services/loaders/channelloader_gen.go
  14. 44
      services/loaders/loaders.go
  15. 19
      services/logs.go
  16. 6
      services/services.go

2
database/mongodb/changes.go

@ -63,7 +63,7 @@ func (r *changeRepository) Insert(ctx context.Context, change models.Change) (*m
change.ID = "Change_" + strconv.Itoa(next)
err = r.changes.Insert(change)
err = r.changes.Insert(&change)
if err != nil {
return nil, err
}

29
database/mongodb/channels.go

@ -7,6 +7,27 @@ import (
"github.com/globalsign/mgo/bson"
)
func newChannelRepository(db *mgo.Database) (*channelRepository, error) {
collection := db.C("common.channels")
err := collection.EnsureIndexKey("logged")
if err != nil {
return nil, err
}
err = collection.EnsureIndexKey("event")
if err != nil {
return nil, err
}
err = collection.EnsureIndexKey("location")
if err != nil {
return nil, err
}
return &channelRepository{
channels: collection,
}, nil
}
type channelRepository struct {
channels *mgo.Collection
}
@ -32,6 +53,9 @@ func (r *channelRepository) List(ctx context.Context, filter models.ChannelFilte
if filter.LocationName != nil {
query["locationName"] = *filter.LocationName
}
if len(filter.Names) > 0 {
query["_id"] = bson.M{"$in": filter.Names}
}
channels := make([]*models.Channel, 0, 32)
err := r.channels.Find(query).Limit(filter.Limit).Sort("_id").All(&channels)
@ -47,7 +71,7 @@ func (r *channelRepository) List(ctx context.Context, filter models.ChannelFilte
}
func (r *channelRepository) Insert(ctx context.Context, channel models.Channel) (*models.Channel, error) {
err := r.channels.Insert(&channel)
err := r.channels.Insert(channel)
if err != nil {
return nil, err
}
@ -59,12 +83,15 @@ func (r *channelRepository) Update(ctx context.Context, channel models.Channel,
updateBson := bson.M{}
if update.Logged != nil {
updateBson["logged"] = *update.Logged
channel.Logged = *update.Logged
}
if update.LocationName != nil {
updateBson["locationName"] = *update.LocationName
channel.LocationName = *update.LocationName
}
if update.EventName != nil {
updateBson["eventName"] = *update.EventName
channel.EventName = *update.EventName
}
err := r.channels.UpdateId(channel.Name, bson.M{"$set": updateBson})

7
database/mongodb/db.go

@ -79,6 +79,12 @@ func Init(cfg config.Database) (*MongoDB, error) {
return nil, err
}
channels, err := newChannelRepository(db)
if err != nil {
session.Close()
return nil, err
}
changes, err := newChangeRepository(db)
if err != nil {
session.Close()
@ -104,6 +110,7 @@ func Init(cfg config.Database) (*MongoDB, error) {
changes: changes,
characters: characters,
channels: channels,
tags: newTagRepository(db),
logs: logs,
posts: posts,

3
graph2/complexity.go

@ -3,7 +3,6 @@ package graph2
import (
"git.aiterp.net/rpdata/api/graph2/graphcore"
"git.aiterp.net/rpdata/api/models"
"git.aiterp.net/rpdata/api/models/channels"
"git.aiterp.net/rpdata/api/models/files"
"git.aiterp.net/rpdata/api/models/stories"
)
@ -25,7 +24,7 @@ func complexity() (cr graphcore.ComplexityRoot) {
cr.Query.Channel = func(childComplexity int, name string) int {
return childComplexity + findComplexity
}
cr.Query.Channels = func(childComplexity int, filter *channels.Filter) int {
cr.Query.Channels = func(childComplexity int, filter *models.ChannelFilter) int {
return childComplexity + listComplexity
}
cr.Query.Post = func(childComplexity int, id string) int {

2
graph2/gqlgen.yml

@ -24,7 +24,7 @@ models:
Channel:
model: git.aiterp.net/rpdata/api/models.Channel
ChannelsFilter:
model: git.aiterp.net/rpdata/api/models/channels.Filter
model: git.aiterp.net/rpdata/api/models.ChannelFilter
UnknownNick:
model: git.aiterp.net/rpdata/api/models.UnknownNick
Post:

79
graph2/resolvers/channel.go

@ -2,93 +2,48 @@ package resolvers
import (
"context"
"errors"
"git.aiterp.net/rpdata/api/graph2/graphcore"
"git.aiterp.net/rpdata/api/internal/auth"
"git.aiterp.net/rpdata/api/models"
"git.aiterp.net/rpdata/api/models/changekeys"
"git.aiterp.net/rpdata/api/models/changes"
"git.aiterp.net/rpdata/api/models/channels"
)
// Queries
func (r *queryResolver) Channel(ctx context.Context, name string) (*models.Channel, error) {
channel, err := channels.FindName(name)
if err != nil {
return nil, err
}
return &channel, nil
return r.s.Channels.Find(ctx, name)
}
func (r *queryResolver) Channels(ctx context.Context, filter *channels.Filter) ([]*models.Channel, error) {
channels, err := channels.List(filter)
if err != nil {
return nil, err
func (r *queryResolver) Channels(ctx context.Context, filter *models.ChannelFilter) ([]*models.Channel, error) {
if filter == nil {
filter = &models.ChannelFilter{}
}
channels2 := make([]*models.Channel, len(channels))
for i := range channels {
channels2[i] = &channels[i]
}
return channels2, nil
return r.s.Channels.List(ctx, *filter)
}
// Mutations
func (r *mutationResolver) AddChannel(ctx context.Context, input graphcore.ChannelAddInput) (*models.Channel, error) {
token := auth.TokenFromContext(ctx)
if !token.Authenticated() || !token.Permitted("channel.add") {
return nil, errors.New("You are not permitted to add channels")
}
logged := input.Logged != nil && *input.Logged
hub := input.Hub != nil && *input.Hub
logged := false
if input.Logged != nil {
logged = *input.Logged
}
hub := false
if input.Hub != nil {
hub = *input.Hub
}
eventName := ""
if input.EventName != nil {
eventName = *input.EventName
}
locationName := ""
if input.LocationName != nil {
locationName = *input.LocationName
}
channel, err := channels.Add(input.Name, logged, hub, eventName, locationName)
if err != nil {
return nil, errors.New("Failed to add channel: " + err.Error())
eventName := ""
if input.LocationName != nil {
eventName = *input.EventName
}
go changes.Submit("Channel", "add", token.UserID, true, changekeys.Listed(channel), channel)
return &channel, nil
return r.s.Channels.Create(ctx, input.Name, logged, hub, eventName, locationName)
}
func (r *mutationResolver) EditChannel(ctx context.Context, input graphcore.ChannelEditInput) (*models.Channel, error) {
token := auth.TokenFromContext(ctx)
if !token.Authenticated() || !token.Permitted("channel.edit") {
return nil, errors.New("You are not permitted to edit channels")
}
channel, err := channels.FindName(input.Name)
if err != nil {
return nil, errors.New("Channel not found")
}
channel, err = channels.Edit(channel, input.Logged, input.Hub, input.EventName, input.LocationName)
if err != nil {
return nil, errors.New("Failed to edit channel: " + err.Error())
}
go changes.Submit("Channel", "edit", token.UserID, true, changekeys.Listed(channel), channel)
return &channel, nil
return r.s.Channels.Update(ctx, input.Name, models.ChannelUpdate{
Logged: input.Logged,
Hub: input.Hub,
EventName: input.EventName,
LocationName: input.LocationName,
})
}

12
graph2/types/log.go

@ -2,25 +2,19 @@ package types
import (
"context"
"errors"
"git.aiterp.net/rpdata/api/services"
"git.aiterp.net/rpdata/api/internal/loader"
"git.aiterp.net/rpdata/api/models"
)
type logResolver struct {
logs *services.LogService
characters *services.CharacterService
channels *services.ChannelService
}
func (r *logResolver) Channel(ctx context.Context, log *models.Log) (*models.Channel, error) {
loader := loader.FromContext(ctx)
if loader == nil {
return nil, errors.New("no loader")
}
return loader.Channel("name", log.ChannelName)
return r.channels.Find(ctx, log.ChannelName)
}
func (r *logResolver) Characters(ctx context.Context, log *models.Log) ([]*models.Character, error) {
@ -39,5 +33,5 @@ func (r *logResolver) Posts(ctx context.Context, log *models.Log, kinds []string
// LogResolver is a resolver
func LogResolver(s *services.Bundle) *logResolver {
return &logResolver{characters: s.Characters, logs: s.Logs}
return &logResolver{characters: s.Characters, logs: s.Logs, channels: s.Channels}
}

16
models/change.go

@ -76,28 +76,28 @@ func (change *Change) Objects() []interface{} {
data := make([]interface{}, 0, 8)
for _, log := range change.Logs {
data = append(data, &log)
data = append(data, log)
}
for _, channel := range change.Channels {
data = append(data, &channel)
data = append(data, channel)
}
for _, character := range change.Characters {
data = append(data, &character)
data = append(data, character)
}
for _, post := range change.Posts {
data = append(data, &post)
data = append(data, post)
}
for _, story := range change.Stories {
data = append(data, &story)
data = append(data, story)
}
for _, tag := range change.Tags {
data = append(data, &tag)
data = append(data, tag)
}
for _, chapter := range change.Chapters {
data = append(data, &chapter)
data = append(data, chapter)
}
for _, comment := range change.Comments {
data = append(data, &comment)
data = append(data, comment)
}
return data

9
models/channel.go

@ -17,10 +17,11 @@ func (*Channel) IsChangeObject() {
// ChannelFilter is a filter for channel listing.
type ChannelFilter struct {
Logged *bool `json:"logged"`
EventName *string `json:"eventName"`
LocationName *string `json:"locationName"`
Limit int `json:"limit"`
Names []string `json:"names"`
Logged *bool `json:"logged"`
EventName *string `json:"eventName"`
LocationName *string `json:"locationName"`
Limit int `json:"limit"`
}
// ChannelUpdate is a filter for channel listing.

5
services/changes.go

@ -42,6 +42,7 @@ func (s *ChangeService) Submit(ctx context.Context, model models.ChangeModel, op
Author: token.UserID,
Listed: listed,
Keys: keys,
Date: time.Now(),
}
for _, obj := range objects {
@ -115,7 +116,9 @@ func (s *ChangeService) loop() {
change, err := s.changes.Insert(timeout, *change)
if err != nil {
log.Println("Failed to submit change:")
log.Println("Failed to insert change:")
} else {
log.Println("Change", change.ID, "inserted.")
}
s.mutex.Lock()

137
services/channels.go

@ -0,0 +1,137 @@
package services
import (
"context"
"git.aiterp.net/rpdata/api/internal/auth"
"git.aiterp.net/rpdata/api/models"
"git.aiterp.net/rpdata/api/models/changekeys"
"git.aiterp.net/rpdata/api/repositories"
"git.aiterp.net/rpdata/api/services/loaders"
)
// ChannelService is an application service for dealing with channels.
type ChannelService struct {
channels repositories.ChannelRepository
loader *loaders.ChannelLoader
changeService *ChangeService
}
func (s *ChannelService) Find(ctx context.Context, id string) (*models.Channel, error) {
return s.loader.Load(id)
}
func (s *ChannelService) List(ctx context.Context, filter models.ChannelFilter) ([]*models.Channel, error) {
if filter.LocationName != nil && filter.EventName != nil && filter.Logged != nil && filter.Limit == 0 && len(filter.Names) > 0 {
channels, errs := s.loader.LoadAll(filter.Names)
for _, err := range errs {
if err != nil && err != repositories.ErrNotFound {
return nil, err
}
}
output := make([]*models.Channel, 0, len(channels))
for _, channel := range channels {
if channel == nil {
continue
}
output = append(output, channel)
}
return output, nil
}
return s.channels.List(ctx, filter)
}
func (s *ChannelService) Create(ctx context.Context, name string, logged, hub bool, eventName, locationName string) (*models.Channel, error) {
err := auth.CheckPermission(ctx, "add", &models.Channel{})
if err != nil {
return nil, err
}
channel, err := s.channels.Insert(ctx, models.Channel{
Name: name,
Logged: logged,
Hub: hub,
EventName: eventName,
LocationName: locationName,
})
if err != nil {
return nil, err
}
s.changeService.Submit(ctx, "Channel", "add", true, changekeys.Listed(channel), channel)
return channel, nil
}
func (s *ChannelService) Ensure(ctx context.Context, name string) (*models.Channel, error) {
err := auth.CheckPermission(ctx, "add", &models.Channel{})
if err != nil {
return nil, err
}
channel, err := s.channels.Find(ctx, name)
if err != nil {
channel, err = s.channels.Insert(ctx, models.Channel{
Name: name,
Logged: false,
Hub: false,
EventName: "",
LocationName: "",
})
if err != nil {
return nil, err
}
s.changeService.Submit(ctx, "Channel", "add", true, changekeys.Listed(channel), channel)
}
return channel, nil
}
func (s *ChannelService) Update(ctx context.Context, name string, update models.ChannelUpdate) (*models.Channel, error) {
channel, err := s.channels.Find(ctx, name)
if err != nil {
return nil, err
}
err = auth.CheckPermission(ctx, "edit", channel)
if err != nil {
return nil, err
}
channel, err = s.channels.Update(ctx, *channel, update)
if err != nil {
return nil, err
}
s.loader.Clear(channel.Name)
s.changeService.Submit(ctx, "Channel", "edit", true, changekeys.Listed(channel), channel)
return channel, nil
}
func (s *ChannelService) Delete(ctx context.Context, name string) (*models.Channel, error) {
channel, err := s.channels.Find(ctx, name)
if err != nil {
return nil, err
}
err = auth.CheckPermission(ctx, "remove", channel)
if err != nil {
return nil, err
}
err = s.channels.Remove(ctx, *channel)
if err != nil {
return nil, err
}
s.changeService.Submit(ctx, "Channel", "remove", true, changekeys.Listed(channel), channel)
s.loader.Clear(channel.Name)
return channel, nil
}

1
services/characters.go

@ -199,7 +199,6 @@ func (s *CharacterService) Delete(ctx context.Context, id string) (*models.Chara
}
s.loader.Clear(character.ID)
s.loader.Prime(character.ID, character)
s.changeService.Submit(ctx, "Character", "remove", true, changekeys.Listed(character), character)

224
services/loaders/channelloader_gen.go

@ -0,0 +1,224 @@
// Code generated by github.com/vektah/dataloaden, DO NOT EDIT.
package loaders
import (
"sync"
"time"
"git.aiterp.net/rpdata/api/models"
)
// ChannelLoaderConfig captures the config to create a new ChannelLoader
type ChannelLoaderConfig struct {
// Fetch is a method that provides the data for the loader
Fetch func(keys []string) ([]*models.Channel, []error)
// Wait is how long wait before sending a batch
Wait time.Duration
// MaxBatch will limit the maximum number of keys to send in one batch, 0 = not limit
MaxBatch int
}
// NewChannelLoader creates a new ChannelLoader given a fetch, wait, and maxBatch
func NewChannelLoader(config ChannelLoaderConfig) *ChannelLoader {
return &ChannelLoader{
fetch: config.Fetch,
wait: config.Wait,
maxBatch: config.MaxBatch,
}
}
// ChannelLoader batches and caches requests
type ChannelLoader struct {
// this method provides the data for the loader
fetch func(keys []string) ([]*models.Channel, []error)
// how long to done before sending a batch
wait time.Duration
// this will limit the maximum number of keys to send in one batch, 0 = no limit
maxBatch int
// INTERNAL
// lazily created cache
cache map[string]*models.Channel
// the current batch. keys will continue to be collected until timeout is hit,
// then everything will be sent to the fetch method and out to the listeners
batch *channelLoaderBatch
// mutex to prevent races
mu sync.Mutex
}
type channelLoaderBatch struct {
keys []string
data []*models.Channel
error []error
closing bool
done chan struct{}
}
// Load a Channel by key, batching and caching will be applied automatically
func (l *ChannelLoader) Load(key string) (*models.Channel, error) {
return l.LoadThunk(key)()
}
// LoadThunk returns a function that when called will block waiting for a Channel.
// This method should be used if you want one goroutine to make requests to many
// different data loaders without blocking until the thunk is called.
func (l *ChannelLoader) LoadThunk(key string) func() (*models.Channel, error) {
l.mu.Lock()
if it, ok := l.cache[key]; ok {
l.mu.Unlock()
return func() (*models.Channel, error) {
return it, nil
}
}
if l.batch == nil {
l.batch = &channelLoaderBatch{done: make(chan struct{})}
}
batch := l.batch
pos := batch.keyIndex(l, key)
l.mu.Unlock()
return func() (*models.Channel, error) {
<-batch.done
var data *models.Channel
if pos < len(batch.data) {
data = batch.data[pos]
}
var err error
// its convenient to be able to return a single error for everything
if len(batch.error) == 1 {
err = batch.error[0]
} else if batch.error != nil {
err = batch.error[pos]
}
if err == nil {
l.mu.Lock()
l.unsafeSet(key, data)
l.mu.Unlock()
}
return data, err
}
}
// LoadAll fetches many keys at once. It will be broken into appropriate sized
// sub batches depending on how the loader is configured
func (l *ChannelLoader) LoadAll(keys []string) ([]*models.Channel, []error) {
results := make([]func() (*models.Channel, error), len(keys))
for i, key := range keys {
results[i] = l.LoadThunk(key)
}
channels := make([]*models.Channel, len(keys))
errors := make([]error, len(keys))
for i, thunk := range results {
channels[i], errors[i] = thunk()
}
return channels, errors
}
// LoadAllThunk returns a function that when called will block waiting for a Channels.
// This method should be used if you want one goroutine to make requests to many
// different data loaders without blocking until the thunk is called.
func (l *ChannelLoader) LoadAllThunk(keys []string) func() ([]*models.Channel, []error) {
results := make([]func() (*models.Channel, error), len(keys))
for i, key := range keys {
results[i] = l.LoadThunk(key)
}
return func() ([]*models.Channel, []error) {
channels := make([]*models.Channel, len(keys))
errors := make([]error, len(keys))
for i, thunk := range results {
channels[i], errors[i] = thunk()
}
return channels, errors
}
}
// Prime the cache with the provided key and value. If the key already exists, no change is made
// and false is returned.
// (To forcefully prime the cache, clear the key first with loader.clear(key).prime(key, value).)
func (l *ChannelLoader) Prime(key string, value *models.Channel) bool {
l.mu.Lock()
var found bool
if _, found = l.cache[key]; !found {
// make a copy when writing to the cache, its easy to pass a pointer in from a loop var
// and end up with the whole cache pointing to the same value.
cpy := *value
l.unsafeSet(key, &cpy)
}
l.mu.Unlock()
return !found
}
// Clear the value at key from the cache, if it exists
func (l *ChannelLoader) Clear(key string) {
l.mu.Lock()
delete(l.cache, key)
l.mu.Unlock()
}
func (l *ChannelLoader) unsafeSet(key string, value *models.Channel) {
if l.cache == nil {
l.cache = map[string]*models.Channel{}
}
l.cache[key] = value
}
// keyIndex will return the location of the key in the batch, if its not found
// it will add the key to the batch
func (b *channelLoaderBatch) keyIndex(l *ChannelLoader, key string) int {
for i, existingKey := range b.keys {
if key == existingKey {
return i
}
}
pos := len(b.keys)
b.keys = append(b.keys, key)
if pos == 0 {
go b.startTimer(l)
}
if l.maxBatch != 0 && pos >= l.maxBatch-1 {
if !b.closing {
b.closing = true
l.batch = nil
go b.end(l)
}
}
return pos
}
func (b *channelLoaderBatch) startTimer(l *ChannelLoader) {
time.Sleep(l.wait)
l.mu.Lock()
// we must have hit a batch limit and are already finalizing this batch
if b.closing {
l.mu.Unlock()
return
}
l.batch = nil
l.mu.Unlock()
b.end(l)
}
func (b *channelLoaderBatch) end(l *ChannelLoader) {
b.data, b.error = l.fetch(b.keys)
close(b.done)
}

44
services/loaders/loaders.go

@ -8,6 +8,7 @@ import (
)
//go:generate go run github.com/vektah/dataloaden CharacterLoader string *git.aiterp.net/rpdata/api/models.Character
//go:generate go run github.com/vektah/dataloaden ChannelLoader string *git.aiterp.net/rpdata/api/models.Channel
// CharacterLoaderFromRepository creates a new CharacterLoader
func CharacterLoaderFromRepository(repo repositories.CharacterRepository) *CharacterLoader {
@ -31,7 +32,6 @@ func CharacterLoaderFromRepository(repo repositories.CharacterRepository) *Chara
}
charMap := make(map[string]*models.Character, len(keys))
for _, character := range characters {
charMap[character.ID] = character
}
@ -51,3 +51,45 @@ func CharacterLoaderFromRepository(repo repositories.CharacterRepository) *Chara
},
}
}
// ChannelLoaderFromRepository creates a new CharacterLoader
func ChannelLoaderFromRepository(repo repositories.ChannelRepository) *ChannelLoader {
return &ChannelLoader{
wait: time.Millisecond * 1,
maxBatch: 100,
fetch: func(keys []string) ([]*models.Channel, []error) {
timeout, cancel := context.WithTimeout(context.Background(), time.Second*15)
defer cancel()
channels, err := repo.List(timeout, models.ChannelFilter{
Names: keys,
})
if err != nil {
errs := make([]error, len(keys))
for i := range errs {
errs[i] = err
}
return nil, errs
}
channelMap := make(map[string]*models.Channel, len(keys))
for _, channel := range channels {
channelMap[channel.Name] = channel
}
results := make([]*models.Channel, len(keys))
errs := make([]error, len(keys))
for i, key := range keys {
if channel, ok := channelMap[key]; ok {
results[i] = channel
} else {
errs[i] = repositories.ErrNotFound
}
}
return results, errs
},
}
}

19
services/logs.go

@ -13,9 +13,10 @@ import (
)
type LogService struct {
logs repositories.LogRepository
posts repositories.PostRepository
changeService *ChangeService
logs repositories.LogRepository
posts repositories.PostRepository
changeService *ChangeService
channelService *ChannelService
}
func (s *LogService) Find(ctx context.Context, id string) (*models.Log, error) {
@ -69,9 +70,12 @@ func (s *LogService) Create(ctx context.Context, title, description, channelName
return nil, err
}
// TODO: s.channelService.Ensure()
_, err := s.channelService.Ensure(ctx, channelName)
if err != nil {
return nil, err
}
log, err := s.logs.Insert(ctx, *log)
log, err = s.logs.Insert(ctx, *log)
if err != nil {
return nil, err
}
@ -94,7 +98,10 @@ func (s *LogService) Import(ctx context.Context, importer models.LogImporter, da
eventName = channel.EventName
}
// TODO: Ensure channel
_, err := s.channelService.Ensure(ctx, channelName)
if err != nil {
return nil, err
}
date = date.In(tz)

6
services/services.go

@ -11,6 +11,7 @@ type Bundle struct {
Characters *CharacterService
Changes *ChangeService
Logs *LogService
Channels *ChannelService
}
// NewBundle creates a new bundle.
@ -25,6 +26,11 @@ func NewBundle(db database.Database) *Bundle {
loader: loaders.CharacterLoaderFromRepository(db.Characters()),
changeService: bundle.Changes,
}
bundle.Channels = &ChannelService{
channels: db.Channels(),
loader: loaders.ChannelLoaderFromRepository(db.Channels()),
changeService: bundle.Changes,
}
bundle.Logs = &LogService{
logs: db.Logs(),
posts: db.Posts(),

Loading…
Cancel
Save