rpdata/api
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

More destruction. Utils too, since they cause weird go mod shenanigans.
the build was successful
Details

Browse Source
This commit is contained in:
gisle
2019-08-26 22:22:05 +02:00
parent e690e7ab97
commit 8a313b3bb2
20 changed files with 0 additions and 874 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/rpdata-as2storyimport/main.go
-139
View File
@@ -1,139 +0,0 @@
package main
import (
"flag"
"fmt"
"log"
"strings"
"time"
"git.aiterp.net/rpdata/api/internal/store"
"git.aiterp.net/rpdata/api/models"
"git.aiterp.net/rpdata/api/models/chapters"
"git.aiterp.net/rpdata/api/models/stories"
_ "github.com/go-sql-driver/mysql"
"github.com/jmoiron/sqlx"
)
var flagHost = flag.String("host", "127.0.0.1:3306", "SQL host")
var flagDB = flag.String("database", "aitestory", "SQL database")
var flagUser = flag.String("user", "aitestory", "SQL user")
var flagPassword = flag.String("password", "", "SQL password")
func main() {
flag.Parse()
db, err := sqlx.Connect("mysql", fmt.Sprintf("%s:%s@(%s)/%s", *flagUser, *flagPassword, *flagHost, *flagDB))
if err != nil {
log.Fatalln(err)
}
err = db.Ping()
if err != nil {
log.Fatalln(err)
}
store.Init()
results := make([]storyResult, 0, 64)
rows, err := db.Queryx("SELECT * FROM page WHERE unlisted=0;")
for rows.Next() {
result := storyResult{}
err := rows.StructScan(&result)
if err != nil {
log.Fatalln(err)
}
results = append(results, result)
}
tagResults := make([]tagResult, 0, 256)
rows, err = db.Queryx("SELECT page_id,type,name FROM page_tag LEFT JOIN tag ON tag_id=tag.id;")
for rows.Next() {
result := tagResult{}
err := rows.StructScan(&result)
if err != nil {
log.Fatalln(err)
}
tagResults = append(tagResults, result)
}
for _, result := range results {
fictionalDate, err := time.Parse("2006-01-02 15:04:05", result.FictionalDate)
if err != nil {
if result.FictionalDate != "0000-00-00 00:00:00" {
log.Fatalln(err)
}
}
if fictionalDate.Year() < 1800 {
fictionalDate = time.Time{}
}
publishDate, err := time.Parse("2006-01-02 15:04:05", result.PublishDate)
if err != nil {
log.Fatalln(err)
}
tags := make([]models.Tag, 0, 8)
for _, tagResult := range tagResults {
if tagResult.PageID == result.ID {
tags = append(tags, models.Tag{Kind: models.TagKind(tagResult.Type), Name: tagResult.Name})
}
}
category := models.StoryCategory(result.Category)
if !category.IsValid() {
log.Println(result.Name, "does not have a valid category:", result.Category)
continue
}
story, err := stories.Add(result.Name, result.Author[5:], category, true, false, tags, publishDate, fictionalDate)
if err != nil {
log.Fatalln(err)
}
// Change the story title
title := result.Name
if strings.HasPrefix(result.Source, "#") {
firstNewline := strings.Index(result.Source, "\n")
result.Name = strings.Replace(result.Source[1:firstNewline], "\r", "", -1)
result.Source = result.Source[firstNewline+1:]
result.Source = strings.Replace(result.Source, "\r\n", "\n", -1)
if strings.HasPrefix(result.Source, "\r") || strings.HasPrefix(result.Source, "\n") {
result.Source = result.Source[1:]
}
}
chapter, err := chapters.Add(story, title, result.Author[5:], result.Source, publishDate, &fictionalDate, models.ChapterCommentModeDisabled)
if err != nil {
log.Fatalln(err)
}
fmt.Println(result.ID, "->", story.ID, chapter.ID)
}
}
type tagResult struct {
PageID string `db:"page_id"`
Type string `db:"type"`
Name string `db:"name"`
}
type storyResult struct {
ID string `db:"id"`
Name string `db:"name"`
Author string `db:"author"`
Category string `db:"category"`
FictionalDate string `db:"fictional_date"`
PublishDate string `db:"publish_date"`
EditDate string `db:"edit_date"`
Unlisted bool `db:"unlisted"`
Dated bool `db:"dated"`
Spesific bool `db:"specific"`
Indexed bool `db:"indexed"`
Published bool `db:"published"`
Type string `db:"type"`
Source string `db:"source"`
Cache string `db:"cache"`
BackgroundURL *string `db:"background_url"`
}
cmd/rpdata-ensurechannels/main.go
-41
View File
@@ -1,41 +0,0 @@
package main
import (
"fmt"
"os"
"git.aiterp.net/rpdata/api/internal/store"
"git.aiterp.net/rpdata/api/models/channels"
"git.aiterp.net/rpdata/api/models/logs"
)
func main() {
err := store.Init()
if err != nil {
fmt.Fprintln(os.Stderr, err)
return
}
logs, err := logs.List(nil)
if err != nil {
fmt.Fprintln(os.Stderr, err)
return
}
added := make(map[string]bool, 1024)
for _, log := range logs {
if added[log.ChannelName] {
continue
}
_, err := channels.Ensure(log.ChannelName, false)
if err != nil {
fmt.Fprintln(os.Stderr, log.ID, err)
continue
}
added[log.ChannelName] = true
fmt.Println(log.ChannelName, "ensured")
}
}
cmd/rpdata-lb2charimport/.gitignore
-3
View File
@@ -1,3 +0,0 @@
characters.json
characters.cson
debug
cmd/rpdata-lb2charimport/data.go
-55
View File
@@ -1,55 +0,0 @@
package main
import (
"encoding/json"
"io"
"strings"
)
type charInfo struct {
Nicks []string `json:"nicks"`
Name string `json:"name"`
Author string `json:"player"`
ShortName string `json:"first"`
}
func load(reader io.Reader) ([]charInfo, error) {
data := make(map[string]interface{})
err := json.NewDecoder(reader).Decode(&data)
if err != nil {
return nil, err
}
links := make(map[string]string, len(data))
infos := make([]charInfo, 0, 64)
for key, value := range data {
if info, ok := value.(map[string]interface{}); ok {
name := info["name"].(string)
author := info["player"].(string)
shortName, ok := info["first"].(string)
if !ok {
shortName = strings.SplitN(name, " ", 2)[0]
}
infos = append(infos, charInfo{
Nicks: []string{key},
Name: name,
Author: author,
ShortName: shortName,
})
} else if nick, ok := value.(string); ok {
links[key] = nick
}
}
for key, value := range links {
for i := range infos {
if infos[i].Nicks[0] == value {
infos[i].Nicks = append(infos[i].Nicks, key)
}
}
}
return infos, nil
}
cmd/rpdata-lb2charimport/main.go
-62
View File
@@ -1,62 +0,0 @@
package main
import (
"flag"
"log"
"os"
"git.aiterp.net/rpdata/api/internal/store"
"git.aiterp.net/rpdata/api/models/characters"
)
var fileName = flag.String("file", "./characters.json", "json file to load")
func main() {
file, err := os.Open(*fileName)
if err != nil {
log.Fatalln("Open:", err)
}
infos, err := load(file)
if err != nil {
log.Fatalln("Parse:", err)
}
err = store.Init()
if err != nil {
log.Fatalln("Store:", err)
}
charsAdded, charsFailed := 0, 0
nicksAdded, nicksFailed := 0, 0
for _, info := range infos {
char, err := characters.Add(info.Nicks[0], info.Name, info.ShortName, info.Author, "")
if err != nil {
log.Println(info.Nicks[0], "failed to insert:", err)
charsFailed++
char, err = characters.FindNick(info.Nicks[0])
if err != nil {
continue
}
} else {
log.Println(info.Nicks[0], "added")
charsAdded++
}
for _, alt := range info.Nicks[1:] {
char, err = characters.AddNick(char, alt)
if err != nil {
log.Println(info.Nicks[0], "failed to add nick", alt, "error:", err)
nicksFailed++
} else {
log.Println(info.Nicks[0], "addded nick", alt)
nicksAdded++
}
}
}
log.Printf("Characters – %d/%d", charsAdded, charsFailed+charsAdded)
log.Printf("Alt. Nicks – %d/%d", nicksAdded, nicksFailed+nicksAdded)
}
cmd/rpdata-lb2logimport/.gitignore
-2
View File
@@ -1,2 +0,0 @@
logs
debug
cmd/rpdata-lb2logimport/line.go
-85
View File
@@ -1,85 +0,0 @@
package main
import (
"bufio"
"errors"
"io"
"strconv"
"strings"
"time"
)
type logLine struct {
Verb string
Args []string
Text string
}
func parseFile(reader io.Reader) []logLine {
bufReader := bufio.NewReader(reader)
results := make([]logLine, 0, 512)
for {
line, err := bufReader.ReadString('\n')
if err == io.EOF {
if len(line) < 2 {
break
}
} else if err != nil {
break
}
if len(line) <= 2 {
continue
}
line = strings.Replace(line, "\n", "", 1)
line = strings.Replace(line, "\r", "", 1)
results = append(results, parseLine(line))
}
return results
}
func parseLine(line string) logLine {
textSplit := strings.SplitN(line, " :", 2)
tokens := strings.Split(textSplit[0], " ")
ll := logLine{
Verb: tokens[0],
}
if len(tokens) > 1 {
ll.Args = tokens[1:]
}
if len(textSplit) > 1 {
ll.Text = textSplit[1]
}
return ll
}
func parseFilename(fname string) (date time.Time, channel string, err error) {
date, err = time.ParseInLocation("2006-01-02_150405", fname[:17], time.Local)
if err != nil {
return
}
ms, err := strconv.Atoi(fname[17:20])
if err != nil {
return
}
date = date.Add(time.Duration(ms) * time.Millisecond)
if len(fname) < 23 {
err = errors.New("filename too short")
return
}
channel = fname[21:]
return
}
cmd/rpdata-lb2logimport/main.go
-110
View File
@@ -1,110 +0,0 @@
package main
import (
"fmt"
"log"
"os"
"path"
"strings"
"time"
"git.aiterp.net/rpdata/api/models/posts"
"git.aiterp.net/rpdata/api/internal/store"
"git.aiterp.net/rpdata/api/models/logs"
)
var prefixReplacer = strings.NewReplacer("+", "", "@", "", "!", "", "%", "")
func main() {
err := store.Init()
if err != nil {
log.Fatalln(err)
}
for _, filepath := range os.Args[1:] {
name := strings.Replace(path.Base(filepath), ".txt", "", 1)
file, err := os.Open(filepath)
if err != nil {
log.Println(filepath, err)
continue
}
logLines := parseFile(file)
file.Close()
// Get title and event
title := ""
event := ""
for _, line := range logLines {
if line.Verb == "TITLE" {
title = line.Text
}
if line.Verb == "TAG" {
event = line.Text
}
}
date, channel, err := parseFilename(name)
if err != nil {
log.Fatalln(err)
}
l, err := logs.Add(date, channel, title, event, "", false)
if err != nil {
log.Println(err)
continue
}
_, err = posts.Add(l, time.Now(), "annotation.info", "rpdata-lb2logimport", "This logfile is imported from aitelogs2 and may contain errors or wrong timestamps.")
if err != nil {
log.Println(err)
}
for _, line := range logLines {
if line.Verb != "CHARS" {
continue
}
_, err = posts.Add(l, l.Date, "chars", prefixReplacer.Replace(line.Args[0]), line.Text)
if err != nil {
log.Println(err)
}
}
for _, line := range logLines {
if line.Verb != "SCENE" && line.Verb != "ACTION" && line.Verb != "TEXT" {
continue
}
postTime, err := time.ParseInLocation("2006-01-02 15:04:05", date.Format("2006-01-02")+" "+line.Args[1], time.Local)
diff := postTime.Sub(date)
if err != nil {
log.Println(err)
continue
}
if diff < 0 {
if diff > -time.Second {
postTime = postTime.Add(diff)
} else {
postTime = postTime.Add(time.Hour * 24)
}
}
if line.Args[0][0] == '=' {
line.Verb = "SCENE"
}
_, err = posts.Add(l, postTime, strings.ToLower(line.Verb), prefixReplacer.Replace(line.Args[0]), line.Text)
if err != nil {
log.Println(err)
}
}
l, err = logs.UpdateCharacters(l, nil)
if err != nil {
log.Println(err)
}
fmt.Println(l.ID, "completed")
}
}
cmd/rpdata-wikifileimport/main.go
-77
View File
@@ -1,77 +0,0 @@
package main
import (
"encoding/json"
"fmt"
"log"
"net/http"
"time"
"git.aiterp.net/rpdata/api/internal/config"
"git.aiterp.net/rpdata/api/internal/store"
"git.aiterp.net/rpdata/api/models/files"
)
func main() {
client := http.Client{Timeout: time.Second * 30}
res, err := client.Get(config.Global().Wiki.URL + "?action=query&list=allimages&ailimit=5&aiprop=dimensions|mime|url|user|timestamp&format=json&ailimit=500")
if err != nil {
log.Fatalln(err)
}
if res.StatusCode != 200 {
log.Fatalln(res.Status)
}
data := ResponseData{}
err = json.NewDecoder(res.Body).Decode(&data)
if err != nil {
log.Fatalln(err)
}
store.Init()
for _, info := range data.Query.Allimages {
existing, err := files.FindName("wiki", info.Name, info.User)
if err == nil && existing.ID != "" {
log.Println("Skipping", info.Name, "because it already exists")
continue
}
ts, err := time.Parse(time.RFC3339, info.Timestamp)
if err != nil {
log.Println("Skipping", info.Name, "error:", err)
continue
}
file, err := files.Insert(info.Name, "wiki", info.Mime, info.User, ts, info.Size, info.URL)
if err != nil {
log.Println("Skipping", info.Name, "error:", err)
continue
}
fmt.Println(file.Name, "inserted ( id:", file.ID, ")")
}
}
// The ResponseData from the wiki looks like this
type ResponseData struct {
Batchcomplete string `json:"batchcomplete"`
Query struct {
Allimages []struct {
Descriptionshorturl string `json:"descriptionshorturl"`
Descriptionurl string `json:"descriptionurl"`
Height int `json:"height"`
Mime string `json:"mime"`
Name string `json:"name"`
Ns int `json:"ns"`
Size int64 `json:"size"`
Timestamp string `json:"timestamp"`
Title string `json:"title"`
URL string `json:"url"`
User string `json:"user"`
Width int `json:"width"`
} `json:"allimages"`
} `json:"query"`
}