fix repository bugs and make text searching work more like it did in the past.

database/postgres/character.go

@@ -18,26 +18,6 @@ type characterRepository struct {
 	db            *sql.DB
 }
 
-func (r *characterRepository) character(row psqlcore.DataCharacter) *models.Character {
-	return &models.Character{
-		ID:          strings.Trim(row.ID, " "),
-		Nicks:       row.Nicks,
-		Name:        row.Name,
-		ShortName:   row.ShortName,
-		Author:      row.Author,
-		Description: row.Description,
-	}
-}
-
-func (r *characterRepository) characters(rows []psqlcore.DataCharacter) []*models.Character {
-	results := make([]*models.Character, 0, len(rows))
-	for _, row := range rows {
-		results = append(results, r.character(row))
-	}
-
-	return results
-}
-
 func (r *characterRepository) Find(ctx context.Context, id string) (*models.Character, error) {
 	row, err := psqlcore.New(r.db).SelectCharacterByID(ctx, id)
 	if err != nil {
@@ -53,7 +33,7 @@ func (r *characterRepository) FindNick(ctx context.Context, nick string) (*model
 		return nil, err
 	}
 
-	return r.character(row), nil
+	return r.character(psqlcore.SelectCharacterByIDRow(row)), nil
 }
 
 func (r *characterRepository) FindName(ctx context.Context, name string) (*models.Character, error) {
@@ -62,7 +42,7 @@ func (r *characterRepository) FindName(ctx context.Context, name string) (*model
 		return nil, err
 	}
 
-	return r.character(row), nil
+	return r.character(psqlcore.SelectCharacterByIDRow(row)), nil
 }
 
 func (r *characterRepository) List(ctx context.Context, filter models.CharacterFilter) ([]*models.Character, error) {
@@ -88,7 +68,7 @@ func (r *characterRepository) List(ctx context.Context, filter models.CharacterF
 	}
 	if filter.Search != nil {
 		params.FilterSearch = true
-		params.Search = *filter.Search
+		params.Search = TSQueryFromSearch(*filter.Search)
 	}
 	if filter.Limit > 0 {
 		params.LimitSize = int32(filter.Limit)
@@ -153,7 +133,12 @@ func (r *characterRepository) Insert(ctx context.Context, character models.Chara
 		return nil, err
 	}
 
-	return &character, tx.Commit()
+	err = tx.Commit()
+	if err != nil {
+		return nil, err
+	}
+
+	return &character, nil
 }
 
 func (r *characterRepository) Update(ctx context.Context, character models.Character, update models.CharacterUpdate) (*models.Character, error) {
@@ -199,11 +184,17 @@ func (r *characterRepository) AddNick(ctx context.Context, character models.Char
 	}
 
 	character.Nicks = append(character.Nicks, nick)
+
+	err = tx.Commit()
+	if err != nil {
+		return nil, err
+	}
+
 	return &character, nil
 }
 
 func (r *characterRepository) RemoveNick(ctx context.Context, character models.Character, nick string) (*models.Character, error) {
-	err := psqlcore.New(r.db).AddCharacterNick(ctx, psqlcore.AddCharacterNickParams{ID: character.ID, Nick: nick})
+	err := psqlcore.New(r.db).RemoveCharacterNick(ctx, psqlcore.RemoveCharacterNickParams{ID: character.ID, Nick: nick})
 	if err != nil {
 		return nil, err
 	}
@@ -221,3 +212,23 @@ func (r *characterRepository) RemoveNick(ctx context.Context, character models.C
 func (r *characterRepository) Delete(ctx context.Context, character models.Character) error {
 	return psqlcore.New(r.db).DeleteCharacter(ctx, character.ID)
 }
+
+func (r *characterRepository) character(row psqlcore.SelectCharacterByIDRow) *models.Character {
+	return &models.Character{
+		ID:          strings.Trim(row.ID, " "),
+		Nicks:       row.Nicks,
+		Name:        row.Name,
+		ShortName:   row.ShortName,
+		Author:      row.Author,
+		Description: row.Description,
+	}
+}
+
+func (r *characterRepository) characters(rows []psqlcore.SelectCharactersRow) []*models.Character {
+	results := make([]*models.Character, 0, len(rows))
+	for _, row := range rows {
+		results = append(results, r.character(psqlcore.SelectCharacterByIDRow(row)))
+	}
+
+	return results
+}

database/postgres/db.go

@@ -30,6 +30,8 @@ func Connect(cfg config.Database) (*DB, error) {
 		return nil, err
 	}
 
+	db.SetMaxIdleConns(32)
+
 	q := psqlcore.New(db)
 	if err := q.EnsureCounter(timeout, "data_character_id"); err != nil {
 		return nil, err

database/postgres/logs.go

@@ -32,23 +32,22 @@ func (r *logRepository) List(ctx context.Context, filter models.LogFilter) ([]*m
 	}
 
 	if filter.Search != nil {
-		ids, err := q.SelectLogIDsFromPostSearch(ctx, *filter.Search)
-		if err != nil {
-			return nil, err
-		}
-
-		params.FilterShortID = true
-		params.ShortIds = ids
+		params.FilterSearch = true
+		params.Search = TSQueryFromSearch(*filter.Search)
 	}
 	if filter.Open != nil {
 		params.FilterOpen = true
 		params.Open = *filter.Open
 	}
-	if filter.Channels != nil {
+	if len(filter.Characters) > 0 {
+		params.FilterCharacterID = true
+		params.CharacterIds = filter.Characters
+	}
+	if len(filter.Channels) > 0 {
 		params.FilterChannelName = true
 		params.ChannelNames = filter.Channels
 	}
-	if filter.Events != nil {
+	if len(filter.Events) > 0 {
 		params.FilterEventName = true
 		params.EventNames = filter.Events
 	}
@@ -106,6 +105,10 @@ func (r *logRepository) Insert(ctx context.Context, log models.Log) (*models.Log
 		}
 	}
 
+	if log.CharacterIDs == nil {
+		log.CharacterIDs = []string{}
+	}
+
 	err = q.InsertLog(ctx, psqlcore.InsertLogParams{
 		ID:           log.ID,
 		ShortID:      log.ShortID,
@@ -172,7 +175,7 @@ func (r *logRepository) Delete(ctx context.Context, log models.Log) error {
 	return tx.Commit()
 }
 
-func (r *logRepository) log(log psqlcore.Log) *models.Log {
+func (r *logRepository) log(log psqlcore.SelectLogRow) *models.Log {
 	return &models.Log{
 		ID:           log.ID,
 		ShortID:      log.ShortID,
@@ -186,10 +189,10 @@ func (r *logRepository) log(log psqlcore.Log) *models.Log {
 	}
 }
 
-func (r *logRepository) logs(logs []psqlcore.Log) []*models.Log {
+func (r *logRepository) logs(logs []psqlcore.SelectLogsRow) []*models.Log {
 	results := make([]*models.Log, 0, len(logs))
 	for _, log := range logs {
-		results = append(results, r.log(log))
+		results = append(results, r.log(psqlcore.SelectLogRow(log)))
 	}
 
 	return results

database/postgres/migrations/20210329190042_alter_log_add_ts_vector.sql

@@ -0,0 +1,9 @@
+-- +goose Up
+-- +goose StatementBegin
+ALTER TABLE log ADD COLUMN "ts_vector" TSVECTOR;
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+ALTER TABLE log DROP COLUMN "ts_vector";
+-- +goose StatementEnd

database/postgres/migrations/20210329190250_create_index_log_search.sql

@@ -0,0 +1,9 @@
+-- +goose Up
+-- +goose StatementBegin
+CREATE INDEX log_index_search ON log USING GIN (ts_vector);
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+DROP INDEX IF EXISTS log_index_search;
+-- +goose StatementEnd

database/postgres/posts.go

@@ -47,7 +47,7 @@ func (r *postRepository) List(ctx context.Context, filter models.PostFilter) ([]
 	}
 	if filter.Search != nil {
 		params.FilterSearch = true
-		params.Search = *filter.Search
+		params.Search = TSQueryFromSearch(*filter.Search)
 	}
 	if filter.Limit > 0 {
 		params.LimitSize = int32(filter.Limit)
@@ -78,12 +78,18 @@ func (r *postRepository) Insert(ctx context.Context, post models.Post) (*models.
 		return nil, err
 	}
 
+	_ = psqlcore.New(r.db).GenerateLogTSVector(ctx, post.LogID)
+
 	post.Position = int(position)
 
 	return &post, nil
 }
 
 func (r *postRepository) InsertMany(ctx context.Context, posts ...*models.Post) ([]*models.Post, error) {
+	if len(posts) == 0 {
+		return []*models.Post{}, nil
+	}
+
 	allowedLogID := ""
 	for _, post := range posts {
 		if allowedLogID == "" {
@@ -119,6 +125,11 @@ func (r *postRepository) InsertMany(ctx context.Context, posts ...*models.Post)
 		post.Position = int(offset) + i
 	}
 
+	err = psqlcore.New(r.db).GenerateLogTSVector(ctx, posts[0].LogID)
+	if err != nil {
+		return nil, err
+	}
+
 	return posts, nil
 }
 
@@ -136,6 +147,8 @@ func (r *postRepository) Update(ctx context.Context, post models.Post, update mo
 		return nil, err
 	}
 
+	_ = psqlcore.New(r.db).GenerateLogTSVector(ctx, post.LogID)
+
 	return &post, nil
 }
 
@@ -206,12 +219,25 @@ func (r *postRepository) Move(ctx context.Context, post models.Post, position in
 		return nil, err
 	}
 
+	positions, err := q.SelectPositionsByLogShortID(ctx, post.LogID)
+	if err != nil {
+		return nil, err
+	}
+	prev := int32(0)
+	for _, pos := range positions {
+		if pos != prev+1 {
+			return nil, errors.New("post discontinuity detected")
+		}
+
+		prev = pos
+	}
+
 	err = tx.Commit()
 	if err != nil {
 		return nil, err
 	}
 
-	return r.posts(posts), nil
+	return r.posts2(posts), nil
 }
 
 func (r *postRepository) Delete(ctx context.Context, post models.Post) error {
@@ -241,10 +267,12 @@ func (r *postRepository) Delete(ctx context.Context, post models.Post) error {
 		return err
 	}
 
+	_ = q.GenerateLogTSVector(ctx, post.LogID)
+
 	return tx.Commit()
 }
 
-func (r *postRepository) post(post psqlcore.LogPost) *models.Post {
+func (r *postRepository) post(post psqlcore.SelectPostRow) *models.Post {
 	return &models.Post{
 		ID:       post.ID,
 		LogID:    post.LogShortID,
@@ -256,10 +284,19 @@ func (r *postRepository) post(post psqlcore.LogPost) *models.Post {
 	}
 }
 
-func (r *postRepository) posts(posts []psqlcore.LogPost) []*models.Post {
+func (r *postRepository) posts(posts []psqlcore.SelectPostsRow) []*models.Post {
 	results := make([]*models.Post, 0, len(posts))
 	for _, post := range posts {
-		results = append(results, r.post(post))
+		results = append(results, r.post(psqlcore.SelectPostRow(post)))
+	}
+
+	return results
+}
+
+func (r *postRepository) posts2(posts []psqlcore.SelectPostsByPositionRangeRow) []*models.Post {
+	results := make([]*models.Post, 0, len(posts))
+	for _, post := range posts {
+		results = append(results, r.post(psqlcore.SelectPostRow(post)))
 	}
 
 	return results

database/postgres/queries/chapters.sql

@@ -3,7 +3,7 @@ SELECT * FROM story_chapter WHERE id=$1::TEXT LIMIT 1;
 
 -- name: SelectChapters :many
 SELECT * FROM story_chapter
-WHERE (sqlx.arg(story_id)::TEXT == '' OR story_id = @story_id::TEXT)
+WHERE (@story_id::TEXT = '' OR story_id = @story_id::TEXT)
 ORDER BY created_date
 LIMIT NULLIF(@limit_size::INT, 0);
 

database/postgres/queries/characters.sql

@@ -1,14 +1,14 @@
 -- name: SelectCharacterByID :one
-SELECT * FROM data_character WHERE id = @id::text;
+SELECT id, nicks, name, short_name, author, description FROM data_character WHERE id = @id::text;
 
 -- name: SelectCharacterByNick :one
-SELECT * FROM data_character WHERE nicks <@ ARRAY[@nick::text];
+SELECT id, nicks, name, short_name, author, description FROM data_character WHERE nicks <@ ARRAY[@nick::text];
 
 -- name: SelectCharacterByName :one
-SELECT * FROM data_character WHERE name = @name::text;
+SELECT id, nicks, name, short_name, author, description FROM data_character WHERE name = @name::text;
 
 -- name: SelectCharacters :many
-SELECT * FROM data_character
+SELECT id, nicks, name, short_name, author, description FROM data_character
 WHERE (@filter_id::bool = false OR id = ANY(@ids::text[]))
     AND (@filter_name::bool = false OR name = ANY(@names::text[]))
     AND (@filter_nick::bool = false OR nicks && (@nicks::text[]))

database/postgres/queries/counter.sql

@@ -5,4 +5,4 @@ INSERT INTO core_counter (id, value) VALUES (@id::text, 0) ON CONFLICT DO NOTHIN
 UPDATE core_counter SET value = value + 1 WHERE id = @id::text RETURNING value::int;
 
 -- name: BumpCounter :exec
-UPDATE core_counter SET value = value + 1 WHERE id = @id::text AND value <= @value::int;
+UPDATE core_counter SET value = @value::int WHERE id = @id::text AND value <= @value::int;

database/postgres/queries/logs.sql

@@ -1,16 +1,17 @@
 -- name: SelectLog :one
-SELECT * FROM log WHERE id=$1 OR short_id=$1 LIMIT 1;
+SELECT id, short_id, character_ids, date, channel_name, event_name, title, description, open FROM log WHERE id=$1 OR short_id=$1 LIMIT 1;
 
 -- name: SelectLogs :many
-SELECT * FROM log
+SELECT id, short_id, character_ids, date, channel_name, event_name, title, description, open FROM log
 WHERE (@filter_short_id::BOOL = false OR short_id = ANY(@short_ids::TEXT[]))
-  AND (@filter_character_id::BOOL = false OR character_ids && (@character_ids::TEXT[]))
+  AND (@filter_character_id::BOOL = false OR character_ids @> (@character_ids::TEXT[]))
   AND (@filter_channel_name::BOOL = false OR channel_name = ANY(@channel_names::TEXT[]))
   AND (@filter_event_name::BOOL = false OR event_name = ANY(@event_names::TEXT[]))
   AND (@filter_open::BOOL = false OR open = @open::BOOL)
   AND (@filter_earlist_date::BOOL = false OR date >= @earliest_date::TIMESTAMP)
   AND (@filter_lastest_date::BOOL = false OR date <= @latest_date::TIMESTAMP)
-ORDER BY date
+  AND (@filter_search::BOOL = false OR "ts_vector" @@ to_tsquery(@search::TEXT))
+ORDER BY date DESC
 LIMIT NULLIF(@limit_size::INT, 0);
 
 -- name: InsertLog :exec
@@ -28,5 +29,16 @@ SET title = @title,
     character_ids = @character_ids
 WHERE id = @id;
 
+-- name: GenerateLogTSVector :exec
+UPDATE log
+SET "ts_vector" = (
+    SELECT TO_TSVECTOR(
+        immutable_array_to_string(array_agg(text), '\n\n')
+    )
+    FROM log_post
+    WHERE log_short_id = @short_id
+)
+WHERE short_id = @short_id;
+
 -- name: DeleteLog :exec
 DELETE FROM log WHERE id=$1;

database/postgres/queries/posts.sql

@@ -1,14 +1,17 @@
 -- name: SelectPost :one
-SELECT * FROM log_post WHERE id=$1 LIMIT 1;
+SELECT id, log_short_id, time, kind, nick, text, position FROM log_post WHERE id=$1 LIMIT 1;
 
 -- name: SelectLogIDsFromPostSearch :many
 SELECT DISTINCT(log_short_id) FROM log_post WHERE "ts_vector" @@ to_tsquery(@search::TEXT);
 
 -- name: SelectPostsByPositionRange :many
-SELECT * FROM log_post WHERE log_short_id = @log_short_id AND position >= @from_position AND position = @to_position;
+SELECT id, log_short_id, time, kind, nick, text, position FROM log_post WHERE log_short_id = @log_short_id AND position >= @from_position AND position <= @to_position;
+
+-- name: SelectPositionsByLogShortID :many
+SELECT position FROM log_post WHERE log_short_id = $1 ORDER BY position;
 
 -- name: SelectPosts :many
-SELECT * FROM log_post
+SELECT id, log_short_id, time, kind, nick, text, position FROM log_post
 WHERE (@filter_ids::BOOL = false OR id = ANY(@ids::TEXT[]))
   AND (@filter_kinds::BOOL = false OR kind = ANY(@kinds::TEXT[]))
   AND (@filter_log_short_id::BOOL = false OR log_short_id = @log_short_id)
@@ -20,13 +23,13 @@ LIMIT NULLIF(@limit_size::INT, 0);
 INSERT INTO log_post (id, log_short_id, time, kind, nick, text, position, ts_vector)
     SELECT @id, @log_short_id, @time, @kind, @nick, @text, COALESCE(MAX(position), 0)+1, to_tsvector(@nick || ' ' || @text)
     FROM log_post
-    WHERE log_short_id=@log_short_id
+    WHERE log_short_id = @log_short_id
 RETURNING position;
 
 -- name: InsertPosts :one
 INSERT INTO log_post (id, log_short_id, time, kind, nick, text, position, ts_vector)
     SELECT UNNEST(@ids::TEXT[]), @log_short_id, UNNEST(@times::TIMESTAMP[]), UNNEST(@kinds::TEXT[]),
-           UNNEST(@nicks::TEXT[]), UNNEST(@texts::TEXT[]), COALESCE(MAX(position), 1) + UNNEST(@offsets::INT[]),
+           UNNEST(@nicks::TEXT[]), UNNEST(@texts::TEXT[]), COALESCE(MAX(position), 0) + UNNEST(@offsets::INT[]),
            to_tsvector(UNNEST(@nicks::TEXT[]) || ' ' || UNNEST(@texts::TEXT[]))
     FROM log_post
     WHERE log_short_id = @log_short_id

database/postgres/queries/stories.sql

@@ -4,13 +4,13 @@ SELECT * FROM story WHERE id = $1 LIMIT 1;
 -- name: SelectStories :many
 SELECT * FROM story
 WHERE (@filter_id::bool = false OR id = ANY(@ids::text[]))
-  AND (@filter_author::bool = false OR name = @author::text)
+  AND (@filter_author::bool = false OR author = @author::text)
   AND (@filter_earlist_fictional_date::bool = false OR fictional_date >= @earliest_fictional_date::timestamp)
   AND (@filter_lastest_fictional_date::bool = false OR fictional_date <= @latest_fictional_date::timestamp)
   AND (@filter_category::bool = false OR category = @category::text)
   AND (@filter_open::bool = false OR open = @open::bool)
-  AND (@filter_unlisted::bool = false OR unlisted = @unlisted::bool)
-ORDER BY updated_date
+  AND (@filter_listed::bool = false OR listed = @listed::bool)
+ORDER BY updated_date DESC
 LIMIT NULLIF(@limit_size::INT, 0);
 
 -- name: InsertStory :exec

database/postgres/queries/tags.sql

@@ -53,7 +53,7 @@ SELECT * FROM common_tag WHERE tag = @tag_name::text ORDER BY tag;
 SELECT * FROM common_tag WHERE tag = ANY(@tag_names::text[]) ORDER BY tag;
 
 -- name: SelectTargetsByTags :many
-SELECT DISTINCT(target_id) FROM common_tag WHERE tag = ANY(@tag_names::text[]) AND target_kind = @target_kind::text ORDER BY tag;
+SELECT DISTINCT target_id FROM common_tag WHERE tag = ANY(@tag_names::text[]) AND target_kind = @target_kind::text;
 
 -- name: SelectTagsByTarget :many
 SELECT * FROM common_tag WHERE target_kind = @target_kind AND target_id = @target_id::text ORDER BY tag;

database/postgres/stories.go

@@ -44,12 +44,12 @@ func (r *storyRepository) List(ctx context.Context, filter models.StoryFilter) (
 			return nil, err
 		}
 
+		params.FilterID = true
+		params.Ids = targets
+
 		if len(params.Ids) == 0 {
 			return []*models.Story{}, nil
 		}
-
-		params.FilterID = true
-		params.Ids = targets
 	}
 	if filter.Author != nil {
 		params.FilterAuthor = true
@@ -72,8 +72,8 @@ func (r *storyRepository) List(ctx context.Context, filter models.StoryFilter) (
 		params.Open = *filter.Open
 	}
 	if filter.Unlisted != nil {
-		params.FilterUnlisted = true
-		params.Unlisted = *filter.Unlisted
+		params.FilterListed = true
+		params.Listed = !*filter.Unlisted
 	}
 	if filter.Limit > 0 {
 		params.LimitSize = int32(filter.Limit)

database/postgres/utils.go

@@ -0,0 +1,56 @@
+package postgres
+
+import "strings"
+
+// TSQueryFromSearch generates a TS query from a typical search string.
+func TSQueryFromSearch(search string) string {
+	if strings.HasPrefix(search, "tsq:") {
+		return search[4:]
+	}
+
+	tokens := strings.Split(search, " ")
+	inQuotes := false
+	result := ""
+
+	for i, token := range tokens {
+		clearQuotes := false
+		startQuotes := false
+
+		if strings.HasPrefix(token, "\"") {
+			token = token[1:]
+			startQuotes = true
+		}
+		if strings.HasSuffix(token, "\"") {
+			token = token[:len(token)-1]
+			clearQuotes = true
+		}
+
+		if startQuotes {
+			if i > 0 {
+				result += " & "
+			}
+
+			result += "("
+		} else if inQuotes {
+			result += "<->"
+		} else if i != 0 {
+			result += " & "
+		}
+
+		result += token
+
+		if startQuotes {
+			inQuotes = true
+		}
+		if clearQuotes {
+			inQuotes = false
+			result += ")"
+		}
+	}
+
+	if inQuotes {
+		return result
+	}
+
+	return strings.TrimLeft(result, " ")
+}

database/postgres/utils_test.go

@@ -0,0 +1,23 @@
+package postgres
+
+import (
+	"fmt"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestTSQueryFromSearch(t *testing.T) {
+	rows := [][2]string{
+		{`asari matron`, `asari & matron`},
+		{`"asari matron"`, `(asari<->matron)`},
+		{`"asari matron" blue`, `(asari<->matron) & blue`},
+		{`"christmas present" "wrapping paper"`, `(christmas<->present) & (wrapping<->paper)`},
+		{`stuff`, `stuff`},
+	}
+
+	for i, row := range rows {
+		t.Run(fmt.Sprintf("Row_%d", i), func(t *testing.T) {
+			assert.Equal(t, row[1], TSQueryFromSearch(row[0]))
+		})
+	}
+}