api/cmd/rpdata-wikifileimport/main.go

package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"time"

	"git.aiterp.net/rpdata/api/internal/config"
	"git.aiterp.net/rpdata/api/internal/store"
	"git.aiterp.net/rpdata/api/model/file"
)

func main() {
	client := http.Client{Timeout: time.Second * 30}

	res, err := client.Get(config.Global().Wiki.URL + "?action=query&list=allimages&ailimit=5&aiprop=dimensions|mime|url|user|timestamp&format=json&ailimit=500")
	if err != nil {
		log.Fatalln(err)
	}

	if res.StatusCode != 200 {
		log.Fatalln(res.Status)
	}

	data := ResponseData{}
	err = json.NewDecoder(res.Body).Decode(&data)
	if err != nil {
		log.Fatalln(err)
	}

	store.Init()

	for _, info := range data.Query.Allimages {
		existing, err := file.FindName("wiki", info.Name, info.User)
		if err == nil && existing.ID != "" {
			log.Println("Skipping", info.Name, "because it already exists")
			continue
		}

		ts, err := time.Parse(time.RFC3339, info.Timestamp)
		if err != nil {
			log.Println("Skipping", info.Name, "error:", err)
			continue
		}

		file, err := file.Insert(info.Name, "wiki", info.Mime, info.User, ts, info.Size, info.URL)
		if err != nil {
			log.Println("Skipping", info.Name, "error:", err)
			continue
		}

		fmt.Println(file.Name, "inserted ( id:", file.ID, ")")
	}
}

// The ResponseData from the wiki looks like this
type ResponseData struct {
	Batchcomplete string `json:"batchcomplete"`
	Query         struct {
		Allimages []struct {
			Descriptionshorturl string `json:"descriptionshorturl"`
			Descriptionurl      string `json:"descriptionurl"`
			Height              int    `json:"height"`
			Mime                string `json:"mime"`
			Name                string `json:"name"`
			Ns                  int    `json:"ns"`
			Size                int64  `json:"size"`
			Timestamp           string `json:"timestamp"`
			Title               string `json:"title"`
			URL                 string `json:"url"`
			User                string `json:"user"`
			Width               int    `json:"width"`
		} `json:"allimages"`
	} `json:"query"`
}