From 18a83073e8187c2b36a85082456ee725b36407f8 Mon Sep 17 00:00:00 2001 From: Gisle Aune Date: Tue, 30 Mar 2021 20:06:16 +0200 Subject: [PATCH] fix repository bugs and make text searching work more like it did in the past. --- cmd/rpdata-server/main.go | 22 ++----- database/postgres/character.go | 61 +++++++++++-------- database/postgres/db.go | 2 + database/postgres/logs.go | 27 ++++---- ...20210329190042_alter_log_add_ts_vector.sql | 9 +++ ...20210329190250_create_index_log_search.sql | 9 +++ database/postgres/posts.go | 47 ++++++++++++-- database/postgres/queries/chapters.sql | 2 +- database/postgres/queries/characters.sql | 8 +-- database/postgres/queries/counter.sql | 2 +- database/postgres/queries/logs.sql | 20 ++++-- database/postgres/queries/posts.sql | 13 ++-- database/postgres/queries/stories.sql | 6 +- database/postgres/queries/tags.sql | 2 +- database/postgres/stories.go | 10 +-- database/postgres/utils.go | 56 +++++++++++++++++ database/postgres/utils_test.go | 23 +++++++ models/tag.go | 2 +- services/characters.go | 18 +++++- services/logs.go | 10 ++- services/services.go | 12 ++-- services/stories.go | 3 + 22 files changed, 271 insertions(+), 93 deletions(-) create mode 100644 database/postgres/migrations/20210329190042_alter_log_add_ts_vector.sql create mode 100644 database/postgres/migrations/20210329190250_create_index_log_search.sql create mode 100644 database/postgres/utils.go create mode 100644 database/postgres/utils_test.go diff --git a/cmd/rpdata-server/main.go b/cmd/rpdata-server/main.go index 0b50e4b..20fd9ec 100644 --- a/cmd/rpdata-server/main.go +++ b/cmd/rpdata-server/main.go @@ -3,20 +3,18 @@ package main import ( "context" "fmt" - "git.aiterp.net/rpdata/api/space" - "log" - "net/http" - "runtime/debug" - "strings" - "git.aiterp.net/rpdata/api/database" "git.aiterp.net/rpdata/api/graph2" "git.aiterp.net/rpdata/api/internal/config" "git.aiterp.net/rpdata/api/internal/instrumentation" "git.aiterp.net/rpdata/api/models" "git.aiterp.net/rpdata/api/services" + "git.aiterp.net/rpdata/api/space" "github.com/99designs/gqlgen/handler" "github.com/prometheus/client_golang/prometheus/promhttp" + "log" + "net/http" + "runtime/debug" ) func main() { @@ -54,11 +52,6 @@ func main() { if err != nil { log.Println("Refresh Characters:", err) } - - err = serviceBundle.Logs.FixImportDateBug(context.Background()) - if err != nil { - log.Println("Fix date bug:", err) - } }() go logListedChanges(serviceBundle.Changes) @@ -112,13 +105,6 @@ func queryHandler(services *services.Bundle) http.HandlerFunc { handler.ResolverMiddleware(instrumentation.ResolverMiddleware()), ) return func(w http.ResponseWriter, r *http.Request) { - // >_> - if strings.HasPrefix(r.Header.Get("Authorization"), "Bearer of the curse") { - w.Header().Set("X-Emerald-Herald", "Seek souls. Larger, more powerful souls.") - w.Header().Add("X-Emerald-Herald", "Seek the king, that is the only way.") - w.Header().Add("X-Emerald-Herald", "Lest this land swallow you whole... As it has so many others.") - } - r = services.Auth.RequestWithToken(r) handler.ServeHTTP(w, r) diff --git a/database/postgres/character.go b/database/postgres/character.go index 9d877b6..a16e001 100644 --- a/database/postgres/character.go +++ b/database/postgres/character.go @@ -18,26 +18,6 @@ type characterRepository struct { db *sql.DB } -func (r *characterRepository) character(row psqlcore.DataCharacter) *models.Character { - return &models.Character{ - ID: strings.Trim(row.ID, " "), - Nicks: row.Nicks, - Name: row.Name, - ShortName: row.ShortName, - Author: row.Author, - Description: row.Description, - } -} - -func (r *characterRepository) characters(rows []psqlcore.DataCharacter) []*models.Character { - results := make([]*models.Character, 0, len(rows)) - for _, row := range rows { - results = append(results, r.character(row)) - } - - return results -} - func (r *characterRepository) Find(ctx context.Context, id string) (*models.Character, error) { row, err := psqlcore.New(r.db).SelectCharacterByID(ctx, id) if err != nil { @@ -53,7 +33,7 @@ func (r *characterRepository) FindNick(ctx context.Context, nick string) (*model return nil, err } - return r.character(row), nil + return r.character(psqlcore.SelectCharacterByIDRow(row)), nil } func (r *characterRepository) FindName(ctx context.Context, name string) (*models.Character, error) { @@ -62,7 +42,7 @@ func (r *characterRepository) FindName(ctx context.Context, name string) (*model return nil, err } - return r.character(row), nil + return r.character(psqlcore.SelectCharacterByIDRow(row)), nil } func (r *characterRepository) List(ctx context.Context, filter models.CharacterFilter) ([]*models.Character, error) { @@ -88,7 +68,7 @@ func (r *characterRepository) List(ctx context.Context, filter models.CharacterF } if filter.Search != nil { params.FilterSearch = true - params.Search = *filter.Search + params.Search = TSQueryFromSearch(*filter.Search) } if filter.Limit > 0 { params.LimitSize = int32(filter.Limit) @@ -153,7 +133,12 @@ func (r *characterRepository) Insert(ctx context.Context, character models.Chara return nil, err } - return &character, tx.Commit() + err = tx.Commit() + if err != nil { + return nil, err + } + + return &character, nil } func (r *characterRepository) Update(ctx context.Context, character models.Character, update models.CharacterUpdate) (*models.Character, error) { @@ -199,11 +184,17 @@ func (r *characterRepository) AddNick(ctx context.Context, character models.Char } character.Nicks = append(character.Nicks, nick) + + err = tx.Commit() + if err != nil { + return nil, err + } + return &character, nil } func (r *characterRepository) RemoveNick(ctx context.Context, character models.Character, nick string) (*models.Character, error) { - err := psqlcore.New(r.db).AddCharacterNick(ctx, psqlcore.AddCharacterNickParams{ID: character.ID, Nick: nick}) + err := psqlcore.New(r.db).RemoveCharacterNick(ctx, psqlcore.RemoveCharacterNickParams{ID: character.ID, Nick: nick}) if err != nil { return nil, err } @@ -221,3 +212,23 @@ func (r *characterRepository) RemoveNick(ctx context.Context, character models.C func (r *characterRepository) Delete(ctx context.Context, character models.Character) error { return psqlcore.New(r.db).DeleteCharacter(ctx, character.ID) } + +func (r *characterRepository) character(row psqlcore.SelectCharacterByIDRow) *models.Character { + return &models.Character{ + ID: strings.Trim(row.ID, " "), + Nicks: row.Nicks, + Name: row.Name, + ShortName: row.ShortName, + Author: row.Author, + Description: row.Description, + } +} + +func (r *characterRepository) characters(rows []psqlcore.SelectCharactersRow) []*models.Character { + results := make([]*models.Character, 0, len(rows)) + for _, row := range rows { + results = append(results, r.character(psqlcore.SelectCharacterByIDRow(row))) + } + + return results +} diff --git a/database/postgres/db.go b/database/postgres/db.go index 0d0e73e..862bc32 100644 --- a/database/postgres/db.go +++ b/database/postgres/db.go @@ -30,6 +30,8 @@ func Connect(cfg config.Database) (*DB, error) { return nil, err } + db.SetMaxIdleConns(32) + q := psqlcore.New(db) if err := q.EnsureCounter(timeout, "data_character_id"); err != nil { return nil, err diff --git a/database/postgres/logs.go b/database/postgres/logs.go index 5676242..4ad2c76 100644 --- a/database/postgres/logs.go +++ b/database/postgres/logs.go @@ -32,23 +32,22 @@ func (r *logRepository) List(ctx context.Context, filter models.LogFilter) ([]*m } if filter.Search != nil { - ids, err := q.SelectLogIDsFromPostSearch(ctx, *filter.Search) - if err != nil { - return nil, err - } - - params.FilterShortID = true - params.ShortIds = ids + params.FilterSearch = true + params.Search = TSQueryFromSearch(*filter.Search) } if filter.Open != nil { params.FilterOpen = true params.Open = *filter.Open } - if filter.Channels != nil { + if len(filter.Characters) > 0 { + params.FilterCharacterID = true + params.CharacterIds = filter.Characters + } + if len(filter.Channels) > 0 { params.FilterChannelName = true params.ChannelNames = filter.Channels } - if filter.Events != nil { + if len(filter.Events) > 0 { params.FilterEventName = true params.EventNames = filter.Events } @@ -106,6 +105,10 @@ func (r *logRepository) Insert(ctx context.Context, log models.Log) (*models.Log } } + if log.CharacterIDs == nil { + log.CharacterIDs = []string{} + } + err = q.InsertLog(ctx, psqlcore.InsertLogParams{ ID: log.ID, ShortID: log.ShortID, @@ -172,7 +175,7 @@ func (r *logRepository) Delete(ctx context.Context, log models.Log) error { return tx.Commit() } -func (r *logRepository) log(log psqlcore.Log) *models.Log { +func (r *logRepository) log(log psqlcore.SelectLogRow) *models.Log { return &models.Log{ ID: log.ID, ShortID: log.ShortID, @@ -186,10 +189,10 @@ func (r *logRepository) log(log psqlcore.Log) *models.Log { } } -func (r *logRepository) logs(logs []psqlcore.Log) []*models.Log { +func (r *logRepository) logs(logs []psqlcore.SelectLogsRow) []*models.Log { results := make([]*models.Log, 0, len(logs)) for _, log := range logs { - results = append(results, r.log(log)) + results = append(results, r.log(psqlcore.SelectLogRow(log))) } return results diff --git a/database/postgres/migrations/20210329190042_alter_log_add_ts_vector.sql b/database/postgres/migrations/20210329190042_alter_log_add_ts_vector.sql new file mode 100644 index 0000000..570910a --- /dev/null +++ b/database/postgres/migrations/20210329190042_alter_log_add_ts_vector.sql @@ -0,0 +1,9 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE log ADD COLUMN "ts_vector" TSVECTOR; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE log DROP COLUMN "ts_vector"; +-- +goose StatementEnd diff --git a/database/postgres/migrations/20210329190250_create_index_log_search.sql b/database/postgres/migrations/20210329190250_create_index_log_search.sql new file mode 100644 index 0000000..15b3369 --- /dev/null +++ b/database/postgres/migrations/20210329190250_create_index_log_search.sql @@ -0,0 +1,9 @@ +-- +goose Up +-- +goose StatementBegin +CREATE INDEX log_index_search ON log USING GIN (ts_vector); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP INDEX IF EXISTS log_index_search; +-- +goose StatementEnd diff --git a/database/postgres/posts.go b/database/postgres/posts.go index 96a3b24..e189f37 100644 --- a/database/postgres/posts.go +++ b/database/postgres/posts.go @@ -47,7 +47,7 @@ func (r *postRepository) List(ctx context.Context, filter models.PostFilter) ([] } if filter.Search != nil { params.FilterSearch = true - params.Search = *filter.Search + params.Search = TSQueryFromSearch(*filter.Search) } if filter.Limit > 0 { params.LimitSize = int32(filter.Limit) @@ -78,12 +78,18 @@ func (r *postRepository) Insert(ctx context.Context, post models.Post) (*models. return nil, err } + _ = psqlcore.New(r.db).GenerateLogTSVector(ctx, post.LogID) + post.Position = int(position) return &post, nil } func (r *postRepository) InsertMany(ctx context.Context, posts ...*models.Post) ([]*models.Post, error) { + if len(posts) == 0 { + return []*models.Post{}, nil + } + allowedLogID := "" for _, post := range posts { if allowedLogID == "" { @@ -119,6 +125,11 @@ func (r *postRepository) InsertMany(ctx context.Context, posts ...*models.Post) post.Position = int(offset) + i } + err = psqlcore.New(r.db).GenerateLogTSVector(ctx, posts[0].LogID) + if err != nil { + return nil, err + } + return posts, nil } @@ -136,6 +147,8 @@ func (r *postRepository) Update(ctx context.Context, post models.Post, update mo return nil, err } + _ = psqlcore.New(r.db).GenerateLogTSVector(ctx, post.LogID) + return &post, nil } @@ -206,12 +219,25 @@ func (r *postRepository) Move(ctx context.Context, post models.Post, position in return nil, err } + positions, err := q.SelectPositionsByLogShortID(ctx, post.LogID) + if err != nil { + return nil, err + } + prev := int32(0) + for _, pos := range positions { + if pos != prev+1 { + return nil, errors.New("post discontinuity detected") + } + + prev = pos + } + err = tx.Commit() if err != nil { return nil, err } - return r.posts(posts), nil + return r.posts2(posts), nil } func (r *postRepository) Delete(ctx context.Context, post models.Post) error { @@ -241,10 +267,12 @@ func (r *postRepository) Delete(ctx context.Context, post models.Post) error { return err } + _ = q.GenerateLogTSVector(ctx, post.LogID) + return tx.Commit() } -func (r *postRepository) post(post psqlcore.LogPost) *models.Post { +func (r *postRepository) post(post psqlcore.SelectPostRow) *models.Post { return &models.Post{ ID: post.ID, LogID: post.LogShortID, @@ -256,10 +284,19 @@ func (r *postRepository) post(post psqlcore.LogPost) *models.Post { } } -func (r *postRepository) posts(posts []psqlcore.LogPost) []*models.Post { +func (r *postRepository) posts(posts []psqlcore.SelectPostsRow) []*models.Post { + results := make([]*models.Post, 0, len(posts)) + for _, post := range posts { + results = append(results, r.post(psqlcore.SelectPostRow(post))) + } + + return results +} + +func (r *postRepository) posts2(posts []psqlcore.SelectPostsByPositionRangeRow) []*models.Post { results := make([]*models.Post, 0, len(posts)) for _, post := range posts { - results = append(results, r.post(post)) + results = append(results, r.post(psqlcore.SelectPostRow(post))) } return results diff --git a/database/postgres/queries/chapters.sql b/database/postgres/queries/chapters.sql index 4c168fe..35422c4 100644 --- a/database/postgres/queries/chapters.sql +++ b/database/postgres/queries/chapters.sql @@ -3,7 +3,7 @@ SELECT * FROM story_chapter WHERE id=$1::TEXT LIMIT 1; -- name: SelectChapters :many SELECT * FROM story_chapter -WHERE (sqlx.arg(story_id)::TEXT == '' OR story_id = @story_id::TEXT) +WHERE (@story_id::TEXT = '' OR story_id = @story_id::TEXT) ORDER BY created_date LIMIT NULLIF(@limit_size::INT, 0); diff --git a/database/postgres/queries/characters.sql b/database/postgres/queries/characters.sql index 58f3a03..85c77c1 100644 --- a/database/postgres/queries/characters.sql +++ b/database/postgres/queries/characters.sql @@ -1,14 +1,14 @@ -- name: SelectCharacterByID :one -SELECT * FROM data_character WHERE id = @id::text; +SELECT id, nicks, name, short_name, author, description FROM data_character WHERE id = @id::text; -- name: SelectCharacterByNick :one -SELECT * FROM data_character WHERE nicks <@ ARRAY[@nick::text]; +SELECT id, nicks, name, short_name, author, description FROM data_character WHERE nicks <@ ARRAY[@nick::text]; -- name: SelectCharacterByName :one -SELECT * FROM data_character WHERE name = @name::text; +SELECT id, nicks, name, short_name, author, description FROM data_character WHERE name = @name::text; -- name: SelectCharacters :many -SELECT * FROM data_character +SELECT id, nicks, name, short_name, author, description FROM data_character WHERE (@filter_id::bool = false OR id = ANY(@ids::text[])) AND (@filter_name::bool = false OR name = ANY(@names::text[])) AND (@filter_nick::bool = false OR nicks && (@nicks::text[])) diff --git a/database/postgres/queries/counter.sql b/database/postgres/queries/counter.sql index 5b0b765..535d046 100644 --- a/database/postgres/queries/counter.sql +++ b/database/postgres/queries/counter.sql @@ -5,4 +5,4 @@ INSERT INTO core_counter (id, value) VALUES (@id::text, 0) ON CONFLICT DO NOTHIN UPDATE core_counter SET value = value + 1 WHERE id = @id::text RETURNING value::int; -- name: BumpCounter :exec -UPDATE core_counter SET value = value + 1 WHERE id = @id::text AND value <= @value::int; \ No newline at end of file +UPDATE core_counter SET value = @value::int WHERE id = @id::text AND value <= @value::int; \ No newline at end of file diff --git a/database/postgres/queries/logs.sql b/database/postgres/queries/logs.sql index ddce16d..6484c3e 100644 --- a/database/postgres/queries/logs.sql +++ b/database/postgres/queries/logs.sql @@ -1,16 +1,17 @@ -- name: SelectLog :one -SELECT * FROM log WHERE id=$1 OR short_id=$1 LIMIT 1; +SELECT id, short_id, character_ids, date, channel_name, event_name, title, description, open FROM log WHERE id=$1 OR short_id=$1 LIMIT 1; -- name: SelectLogs :many -SELECT * FROM log +SELECT id, short_id, character_ids, date, channel_name, event_name, title, description, open FROM log WHERE (@filter_short_id::BOOL = false OR short_id = ANY(@short_ids::TEXT[])) - AND (@filter_character_id::BOOL = false OR character_ids && (@character_ids::TEXT[])) + AND (@filter_character_id::BOOL = false OR character_ids @> (@character_ids::TEXT[])) AND (@filter_channel_name::BOOL = false OR channel_name = ANY(@channel_names::TEXT[])) AND (@filter_event_name::BOOL = false OR event_name = ANY(@event_names::TEXT[])) AND (@filter_open::BOOL = false OR open = @open::BOOL) AND (@filter_earlist_date::BOOL = false OR date >= @earliest_date::TIMESTAMP) AND (@filter_lastest_date::BOOL = false OR date <= @latest_date::TIMESTAMP) -ORDER BY date + AND (@filter_search::BOOL = false OR "ts_vector" @@ to_tsquery(@search::TEXT)) +ORDER BY date DESC LIMIT NULLIF(@limit_size::INT, 0); -- name: InsertLog :exec @@ -28,5 +29,16 @@ SET title = @title, character_ids = @character_ids WHERE id = @id; +-- name: GenerateLogTSVector :exec +UPDATE log +SET "ts_vector" = ( + SELECT TO_TSVECTOR( + immutable_array_to_string(array_agg(text), '\n\n') + ) + FROM log_post + WHERE log_short_id = @short_id +) +WHERE short_id = @short_id; + -- name: DeleteLog :exec DELETE FROM log WHERE id=$1; diff --git a/database/postgres/queries/posts.sql b/database/postgres/queries/posts.sql index 7640174..8fdd8f4 100644 --- a/database/postgres/queries/posts.sql +++ b/database/postgres/queries/posts.sql @@ -1,14 +1,17 @@ -- name: SelectPost :one -SELECT * FROM log_post WHERE id=$1 LIMIT 1; +SELECT id, log_short_id, time, kind, nick, text, position FROM log_post WHERE id=$1 LIMIT 1; -- name: SelectLogIDsFromPostSearch :many SELECT DISTINCT(log_short_id) FROM log_post WHERE "ts_vector" @@ to_tsquery(@search::TEXT); -- name: SelectPostsByPositionRange :many -SELECT * FROM log_post WHERE log_short_id = @log_short_id AND position >= @from_position AND position = @to_position; +SELECT id, log_short_id, time, kind, nick, text, position FROM log_post WHERE log_short_id = @log_short_id AND position >= @from_position AND position <= @to_position; + +-- name: SelectPositionsByLogShortID :many +SELECT position FROM log_post WHERE log_short_id = $1 ORDER BY position; -- name: SelectPosts :many -SELECT * FROM log_post +SELECT id, log_short_id, time, kind, nick, text, position FROM log_post WHERE (@filter_ids::BOOL = false OR id = ANY(@ids::TEXT[])) AND (@filter_kinds::BOOL = false OR kind = ANY(@kinds::TEXT[])) AND (@filter_log_short_id::BOOL = false OR log_short_id = @log_short_id) @@ -20,13 +23,13 @@ LIMIT NULLIF(@limit_size::INT, 0); INSERT INTO log_post (id, log_short_id, time, kind, nick, text, position, ts_vector) SELECT @id, @log_short_id, @time, @kind, @nick, @text, COALESCE(MAX(position), 0)+1, to_tsvector(@nick || ' ' || @text) FROM log_post - WHERE log_short_id=@log_short_id + WHERE log_short_id = @log_short_id RETURNING position; -- name: InsertPosts :one INSERT INTO log_post (id, log_short_id, time, kind, nick, text, position, ts_vector) SELECT UNNEST(@ids::TEXT[]), @log_short_id, UNNEST(@times::TIMESTAMP[]), UNNEST(@kinds::TEXT[]), - UNNEST(@nicks::TEXT[]), UNNEST(@texts::TEXT[]), COALESCE(MAX(position), 1) + UNNEST(@offsets::INT[]), + UNNEST(@nicks::TEXT[]), UNNEST(@texts::TEXT[]), COALESCE(MAX(position), 0) + UNNEST(@offsets::INT[]), to_tsvector(UNNEST(@nicks::TEXT[]) || ' ' || UNNEST(@texts::TEXT[])) FROM log_post WHERE log_short_id = @log_short_id diff --git a/database/postgres/queries/stories.sql b/database/postgres/queries/stories.sql index eb7db8a..77270f9 100644 --- a/database/postgres/queries/stories.sql +++ b/database/postgres/queries/stories.sql @@ -4,13 +4,13 @@ SELECT * FROM story WHERE id = $1 LIMIT 1; -- name: SelectStories :many SELECT * FROM story WHERE (@filter_id::bool = false OR id = ANY(@ids::text[])) - AND (@filter_author::bool = false OR name = @author::text) + AND (@filter_author::bool = false OR author = @author::text) AND (@filter_earlist_fictional_date::bool = false OR fictional_date >= @earliest_fictional_date::timestamp) AND (@filter_lastest_fictional_date::bool = false OR fictional_date <= @latest_fictional_date::timestamp) AND (@filter_category::bool = false OR category = @category::text) AND (@filter_open::bool = false OR open = @open::bool) - AND (@filter_unlisted::bool = false OR unlisted = @unlisted::bool) -ORDER BY updated_date + AND (@filter_listed::bool = false OR listed = @listed::bool) +ORDER BY updated_date DESC LIMIT NULLIF(@limit_size::INT, 0); -- name: InsertStory :exec diff --git a/database/postgres/queries/tags.sql b/database/postgres/queries/tags.sql index 943df33..03877a2 100644 --- a/database/postgres/queries/tags.sql +++ b/database/postgres/queries/tags.sql @@ -53,7 +53,7 @@ SELECT * FROM common_tag WHERE tag = @tag_name::text ORDER BY tag; SELECT * FROM common_tag WHERE tag = ANY(@tag_names::text[]) ORDER BY tag; -- name: SelectTargetsByTags :many -SELECT DISTINCT(target_id) FROM common_tag WHERE tag = ANY(@tag_names::text[]) AND target_kind = @target_kind::text ORDER BY tag; +SELECT DISTINCT target_id FROM common_tag WHERE tag = ANY(@tag_names::text[]) AND target_kind = @target_kind::text; -- name: SelectTagsByTarget :many SELECT * FROM common_tag WHERE target_kind = @target_kind AND target_id = @target_id::text ORDER BY tag; diff --git a/database/postgres/stories.go b/database/postgres/stories.go index a86ec6e..7e338c7 100644 --- a/database/postgres/stories.go +++ b/database/postgres/stories.go @@ -44,12 +44,12 @@ func (r *storyRepository) List(ctx context.Context, filter models.StoryFilter) ( return nil, err } + params.FilterID = true + params.Ids = targets + if len(params.Ids) == 0 { return []*models.Story{}, nil } - - params.FilterID = true - params.Ids = targets } if filter.Author != nil { params.FilterAuthor = true @@ -72,8 +72,8 @@ func (r *storyRepository) List(ctx context.Context, filter models.StoryFilter) ( params.Open = *filter.Open } if filter.Unlisted != nil { - params.FilterUnlisted = true - params.Unlisted = *filter.Unlisted + params.FilterListed = true + params.Listed = !*filter.Unlisted } if filter.Limit > 0 { params.LimitSize = int32(filter.Limit) diff --git a/database/postgres/utils.go b/database/postgres/utils.go new file mode 100644 index 0000000..edd630b --- /dev/null +++ b/database/postgres/utils.go @@ -0,0 +1,56 @@ +package postgres + +import "strings" + +// TSQueryFromSearch generates a TS query from a typical search string. +func TSQueryFromSearch(search string) string { + if strings.HasPrefix(search, "tsq:") { + return search[4:] + } + + tokens := strings.Split(search, " ") + inQuotes := false + result := "" + + for i, token := range tokens { + clearQuotes := false + startQuotes := false + + if strings.HasPrefix(token, "\"") { + token = token[1:] + startQuotes = true + } + if strings.HasSuffix(token, "\"") { + token = token[:len(token)-1] + clearQuotes = true + } + + if startQuotes { + if i > 0 { + result += " & " + } + + result += "(" + } else if inQuotes { + result += "<->" + } else if i != 0 { + result += " & " + } + + result += token + + if startQuotes { + inQuotes = true + } + if clearQuotes { + inQuotes = false + result += ")" + } + } + + if inQuotes { + return result + } + + return strings.TrimLeft(result, " ") +} diff --git a/database/postgres/utils_test.go b/database/postgres/utils_test.go new file mode 100644 index 0000000..ffaed02 --- /dev/null +++ b/database/postgres/utils_test.go @@ -0,0 +1,23 @@ +package postgres + +import ( + "fmt" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestTSQueryFromSearch(t *testing.T) { + rows := [][2]string{ + {`asari matron`, `asari & matron`}, + {`"asari matron"`, `(asari<->matron)`}, + {`"asari matron" blue`, `(asari<->matron) & blue`}, + {`"christmas present" "wrapping paper"`, `(christmas<->present) & (wrapping<->paper)`}, + {`stuff`, `stuff`}, + } + + for i, row := range rows { + t.Run(fmt.Sprintf("Row_%d", i), func(t *testing.T) { + assert.Equal(t, row[1], TSQueryFromSearch(row[0])) + }) + } +} diff --git a/models/tag.go b/models/tag.go index 93c7209..14cfebf 100644 --- a/models/tag.go +++ b/models/tag.go @@ -20,7 +20,7 @@ func (tag *Tag) Decode(v string) error { } kind := TagKind("") - err := kind.UnmarshalGQL(v) + err := kind.UnmarshalGQL(split[0]) if err != nil { return err } diff --git a/services/characters.go b/services/characters.go index e9aa90d..02de943 100644 --- a/services/characters.go +++ b/services/characters.go @@ -74,9 +74,23 @@ func (s *CharacterService) List(ctx context.Context, filter models.CharacterFilt return characters, nil } - go s.refreshLogs() + characters, err := s.characters.List(ctx, filter) + if err != nil { + return nil, err + } + + sort.Slice(characters, func(i, j int) bool { + if len(characters[i].ID) > len(characters[j].ID) { + return false + } + if len(characters[i].ID) < len(characters[j].ID) { + return true + } + + return strings.Compare(characters[i].ID, characters[j].ID) < 0 + }) - return s.characters.List(ctx, filter) + return characters, nil } func (s *CharacterService) Create(ctx context.Context, nick, name, shortName, author, description string) (*models.Character, error) { diff --git a/services/logs.go b/services/logs.go index c133215..8490497 100644 --- a/services/logs.go +++ b/services/logs.go @@ -718,11 +718,19 @@ func (s *LogService) RefreshAllLogCharacters(ctx context.Context) error { } s.unknownNicksMutex.Unlock() + tokens := make(chan struct{}, 33) + for i := 0; i < 32; i++ { + tokens <- struct{}{} + } + eg := errgroup.Group{} for i := range logs { l := logs[i] eg.Go(func() error { + <-tokens + defer func() { tokens <- struct{}{} }() + _, err := s.refreshLogCharacters(ctx, *l, characterMap, true) return err }) @@ -748,7 +756,7 @@ func (s *LogService) RefreshLogCharacters(ctx context.Context, log models.Log) ( func (s *LogService) refreshLogCharacters(ctx context.Context, log models.Log, characterMap map[string]*models.Character, useUnknownNicks bool) (*models.Log, error) { posts, err := s.ListPosts(ctx, &models.PostFilter{LogID: &log.ShortID}) if err != nil { - return &log, nil + return nil, err } counts := make(map[string]int) diff --git a/services/services.go b/services/services.go index 5b6fd1d..5dfd452 100644 --- a/services/services.go +++ b/services/services.go @@ -29,11 +29,13 @@ func NewBundle(db database.Database, spaceClient *space.Client) *Bundle { changes: db.Changes(), authService: bundle.Auth, } - bundle.Files = &FileService{ - files: db.Files(), - authService: bundle.Auth, - changeService: bundle.Changes, - space: spaceClient, + if spaceClient != nil { + bundle.Files = &FileService{ + files: db.Files(), + authService: bundle.Auth, + changeService: bundle.Changes, + space: spaceClient, + } } bundle.Tags = &TagService{tags: db.Tags()} bundle.Characters = &CharacterService{ diff --git a/services/stories.go b/services/stories.go index 9d173f3..ccd75ac 100644 --- a/services/stories.go +++ b/services/stories.go @@ -46,6 +46,9 @@ func (s *StoryService) ListStories(ctx context.Context, filter models.StoryFilte return nil, errors.New("you cannot view your own unlisted stories") } } + } else { + unlistedValue := false + filter.Unlisted = &unlistedValue } return s.stories.List(ctx, filter)