packages.wenpai.net/internal/http/handlers.go
elementary 741755ac9b
Some checks failed
ci / GolangCI-Lint (push) Failing after 7m10s
ci / Go Modules Tidy (push) Failing after 5m26s
ci / Go Test (push) Failing after 1h3m11s
ci / Integration Test (push) Failing after 5s
govulncheck / govulncheck (push) Failing after 1s
refactor: mirror-mode cleanups — upstream monthly installs, remove untagged, simplify status
2026-04-12 11:14:17 +08:00

720 lines
20 KiB
Go

package http
import (
"context"
"crypto/sha256"
"database/sql"
"encoding/hex"
"fmt"
"net/http"
"os"
"path/filepath"
"slices"
"strconv"
"strings"
"time"
"github.com/getsentry/sentry-go"
"github.com/roots/wp-packages/internal/app"
"github.com/roots/wp-packages/internal/config"
"github.com/roots/wp-packages/internal/og"
"github.com/roots/wp-packages/internal/telemetry"
"github.com/roots/wp-packages/internal/version"
)
const perPage = 12
// captureError reports a non-panic error to Sentry with the request's hub.
// It silently ignores context cancellation errors (timeouts, client disconnects)
// since these are expected during normal operation.
func captureError(r *http.Request, err error) {
if r.Context().Err() != nil {
return
}
if hub := sentry.GetHubFromContext(r.Context()); hub != nil {
hub.CaptureException(err)
} else {
sentry.CaptureException(err)
}
}
// setETag computes an ETag from the given seed strings, sets it on the
// response, and returns true if the client already has this version (304).
func setETag(w http.ResponseWriter, r *http.Request, parts ...string) bool {
h := sha256.New()
for _, p := range parts {
h.Write([]byte(p))
}
etag := `"` + hex.EncodeToString(h.Sum(nil))[:16] + `"`
w.Header().Set("ETag", etag)
if r.Header.Get("If-None-Match") == etag {
w.WriteHeader(http.StatusNotModified)
return true
}
return false
}
type packageRow struct {
ID int64
Type string
Name string
DisplayName string
Description string
Author string
Homepage string
CurrentVersion string
WporgVersion string
Downloads int64
ActiveInstalls int64
IsActive bool
LastSyncedAt string
LastCommitted string
WpPackagesInstallsTotal int64
}
type versionRow struct {
Version string
IsLatest bool
}
func handleIndex(a *app.App, tmpl *templateSet) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
filters := publicFilters{
Search: r.URL.Query().Get("search"),
Type: r.URL.Query().Get("type"),
Sort: r.URL.Query().Get("sort"),
}
if filters.Sort == "" {
filters.Sort = "composer_installs"
}
page, _ := strconv.Atoi(r.URL.Query().Get("page"))
if page < 1 {
page = 1
}
packages, total, err := queryPackages(r.Context(), a.DB, filters, page, perPage)
if err != nil {
a.Logger.Error("querying packages", "error", err)
captureError(r, err)
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
totalPages := (total + perPage - 1) / perPage
dashStats := queryDashboardStats(r.Context(), a.DB)
s := dashStats["Stats"].(struct {
TotalPackages int64
ActivePlugins int64
ActiveThemes int64
TotalInstalls int64
PluginInstalls int64
ThemeInstalls int64
Installs30d int64
CurrentBuild string
StatsUpdatedAt string
})
updatedAt := s.StatsUpdatedAt
searchURL := a.Config.AppURL + "/?search={search_term_string}"
jsonLDData := map[string]any{
"@context": "https://schema.org",
"@type": "WebSite",
"name": "WP Packages",
"url": a.Config.AppURL + "/",
"potentialAction": map[string]any{
"@type": "SearchAction",
"target": searchURL,
"query-input": "required name=search_term_string",
},
}
w.Header().Set("Cache-Control", "public, max-age=60, s-maxage=300, stale-while-revalidate=3600")
if setETag(w, r, "index", updatedAt, filters.Search, filters.Type, filters.Sort, strconv.Itoa(page)) {
return
}
render(w, r, tmpl.index, "layout", map[string]any{
"Packages": packages,
"Filters": filters,
"Page": page,
"Total": total,
"TotalPages": totalPages,
"Stats": dashStats["Stats"],
"AppURL": a.Config.AppURL,
"CDNURL": a.Config.R2.CDNPublicURL,
"OGImage": ogImageURL(a.Config, "social/default.png"),
"JSONLD": jsonLDData,
"BlogPosts": a.Blog.Posts(),
})
}
}
func handleIndexPartial(a *app.App, tmpl *templateSet) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
filters := publicFilters{
Search: r.URL.Query().Get("search"),
Type: r.URL.Query().Get("type"),
Sort: r.URL.Query().Get("sort"),
}
if filters.Sort == "" {
filters.Sort = "composer_installs"
}
page, _ := strconv.Atoi(r.URL.Query().Get("page"))
if page < 1 {
page = 1
}
packages, total, err := queryPackages(r.Context(), a.DB, filters, page, perPage)
if err != nil {
a.Logger.Error("querying packages", "error", err)
captureError(r, err)
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
totalPages := (total + perPage - 1) / perPage
w.Header().Set("X-Robots-Tag", "noindex")
render(w, r, tmpl.indexPartial, "package-results", map[string]any{
"Packages": packages,
"Filters": filters,
"Page": page,
"Total": total,
"TotalPages": totalPages,
})
}
}
func handleDocs(a *app.App, tmpl *templateSet) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Cache-Control", "public, max-age=3600, stale-while-revalidate=86400")
render(w, r, tmpl.docs, "layout", map[string]any{
"AppURL": a.Config.AppURL,
"CDNURL": a.Config.R2.CDNPublicURL,
"OGImage": ogImageURL(a.Config, "social/default.png"),
})
}
}
func handleCompare(a *app.App, tmpl *templateSet) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Cache-Control", "public, max-age=3600, stale-while-revalidate=86400")
render(w, r, tmpl.compare, "layout", map[string]any{
"AppURL": a.Config.AppURL,
"CDNURL": a.Config.R2.CDNPublicURL,
"OGImage": ogImageURL(a.Config, "social/default.png"),
})
}
}
func handleWordpressCore(a *app.App, tmpl *templateSet) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Cache-Control", "public, max-age=3600, stale-while-revalidate=86400")
render(w, r, tmpl.wordpressCore, "layout", map[string]any{
"AppURL": a.Config.AppURL,
"CDNURL": a.Config.R2.CDNPublicURL,
"OGImage": ogImageURL(a.Config, "social/default.png"),
"RootsDownloads": a.Packagist.Total(),
})
}
}
func handleDetail(a *app.App, tmpl *templateSet) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
pkgType := r.PathValue("type")
name := r.PathValue("name")
// Strip wp- prefix from type
pkgType = strings.TrimPrefix(pkgType, "wp-")
pkg, err := queryPackageDetail(r.Context(), a.DB, pkgType, name)
if err != nil {
gone := packageExistsInactive(r.Context(), a.DB, pkgType, name)
if gone {
http.Redirect(w, r, "https://wp-packages.org/", http.StatusFound)
} else {
w.WriteHeader(http.StatusNotFound)
render(w, r, tmpl.notFound, "layout", map[string]any{"Gone": false, "CDNURL": a.Config.R2.CDNPublicURL})
}
return
}
upstreamVers, _ := fetchUpstreamVersions(pkg.Type, pkg.Name)
versions := make([]versionRow, 0, len(upstreamVers))
for _, v := range upstreamVers {
versions = append(versions, versionRow{
Version: v,
IsLatest: v == pkg.CurrentVersion,
})
}
// Sort by semver descending, with latest always first
slices.SortFunc(versions, func(a, b versionRow) int {
if a.IsLatest != b.IsLatest {
if a.IsLatest {
return -1
}
return 1
}
return version.Compare(b.Version, a.Version)
})
monthlyInstalls, _ := telemetry.GetMonthlyInstalls(r.Context(), a.DB, pkg.ID)
untagged := pkg.Type == "plugin" && pkg.WporgVersion != "" && !versionIsTagged(versions, pkg.WporgVersion)
trunkOnly := untagged && !hasTaggedVersion(versions)
var ogImage string
if pkg.OGImageGeneratedAt != nil {
ogImage = ogImageURL(a.Config, "social/"+pkg.Type+"/"+pkg.Name+".png")
}
displayName := pkg.Name
if pkg.DisplayName != "" {
displayName = pkg.DisplayName
}
pkgURL := a.Config.AppURL + "/packages/wp-" + pkg.Type + "/" + pkg.Name
softwareApp := map[string]any{
"@context": "https://schema.org",
"@type": "SoftwareApplication",
"name": displayName,
"applicationCategory": "WordPress " + pkg.Type,
"operatingSystem": "WordPress",
"url": pkgURL,
}
if pkg.CurrentVersion != "" {
softwareApp["softwareVersion"] = pkg.CurrentVersion
}
if pkg.Author != "" {
softwareApp["author"] = map[string]any{
"@type": "Person",
"name": pkg.Author,
}
}
if pkg.ActiveInstalls > 0 {
softwareApp["interactionStatistic"] = map[string]any{
"@type": "InteractionCounter",
"interactionType": "https://schema.org/InstallAction",
"userInteractionCount": pkg.ActiveInstalls,
}
}
if pkg.Description != "" {
softwareApp["description"] = pkg.Description
}
if pkg.UpdatedAt != "" {
softwareApp["dateModified"] = pkg.UpdatedAt
}
breadcrumbs := map[string]any{
"@context": "https://schema.org",
"@type": "BreadcrumbList",
"itemListElement": []map[string]any{
{
"@type": "ListItem",
"position": 1,
"name": "Packages",
"item": a.Config.AppURL + "/",
},
{
"@type": "ListItem",
"position": 2,
"name": displayName,
"item": pkgURL,
},
},
}
w.Header().Set("Cache-Control", "public, max-age=60, s-maxage=3600, stale-while-revalidate=86400")
if setETag(w, r, "detail", pkg.Type, pkg.Name, pkg.UpdatedAt) {
return
}
render(w, r, tmpl.detail, "layout", map[string]any{
"Package": pkg,
"Versions": versions,
"MonthlyInstalls": monthlyInstalls,
"Untagged": untagged,
"TrunkOnly": trunkOnly,
"AppURL": a.Config.AppURL,
"CDNURL": a.Config.R2.CDNPublicURL,
"OGImage": ogImage,
"JSONLD": []any{softwareApp, breadcrumbs},
})
}
}
var logFiles = map[string]string{
"wppackages": filepath.Join("storage", "logs", "wppackages.log"),
"pipeline": filepath.Join("storage", "logs", "pipeline.log"),
"check-status": filepath.Join("storage", "logs", "check-status.log"),
}
func handleAdminLogs(tmpl *templateSet) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
render(w, r, tmpl.adminLogs, "admin_layout", nil)
}
}
func handleAdminLogStream(a *app.App) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
file := r.URL.Query().Get("file")
logPath, ok := logFiles[file]
if !ok {
http.Error(w, "unknown log file", http.StatusBadRequest)
return
}
flusher, ok := w.(http.Flusher)
if !ok {
http.Error(w, "streaming not supported", http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "text/event-stream")
w.Header().Set("Cache-Control", "no-cache")
w.Header().Set("Connection", "keep-alive")
w.Header().Set("X-Accel-Buffering", "no")
ctx := r.Context()
ticker := time.NewTicker(500 * time.Millisecond)
defer ticker.Stop()
// Wait for the file to exist
var f *os.File
for f == nil {
if opened, err := os.Open(logPath); err == nil {
f = opened
} else {
_, _ = fmt.Fprintf(w, "data: Waiting for %s ...\n\n", filepath.Base(logPath))
flusher.Flush()
select {
case <-ctx.Done():
return
case <-time.After(2 * time.Second):
}
}
}
defer func() { _ = f.Close() }()
// Send initial batch: last 200 lines
lines := tailFile(logPath, 200)
for _, line := range lines {
_, _ = fmt.Fprintf(w, "data: %s\n\n", line)
}
flusher.Flush()
// Seek to end for tailing
offset, _ := f.Seek(0, 2)
buf := make([]byte, 64*1024)
var partial string
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
info, err := os.Stat(logPath)
if err != nil {
continue
}
if info.Size() < offset {
// File was truncated/rotated, reopen
_ = f.Close()
f, err = os.Open(logPath)
if err != nil {
continue
}
offset = 0
}
if info.Size() == offset {
continue
}
_, _ = f.Seek(offset, 0)
n, err := f.Read(buf)
if n > 0 {
offset += int64(n)
chunk := partial + string(buf[:n])
partial = ""
newLines := strings.Split(chunk, "\n")
if !strings.HasSuffix(chunk, "\n") {
partial = newLines[len(newLines)-1]
newLines = newLines[:len(newLines)-1]
}
for _, line := range newLines {
if line != "" {
if _, werr := fmt.Fprintf(w, "data: %s\n\n", line); werr != nil {
return
}
}
}
flusher.Flush()
}
if err != nil {
continue
}
}
}
}
}
func tailFile(path string, n int) []string {
data, err := os.ReadFile(path)
if err != nil {
return nil
}
lines := strings.Split(strings.TrimRight(string(data), "\n"), "\n")
if len(lines) > n {
lines = lines[len(lines)-n:]
}
return lines
}
// ogImageURL returns the full OG image URL for a given key.
// In production, it uses the CDN. In local dev, it uses /og/ routes.
func ogImageURL(cfg *config.Config, key string) string {
if cfg.R2.CDNPublicURL != "" {
return cfg.R2.CDNPublicURL + "/" + key
}
if cfg.AppURL != "" {
return cfg.AppURL + "/og/" + key
}
return "/og/" + key
}
// handleOGImage serves OG images from local disk (dev mode).
func handleOGImage() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
key := strings.TrimPrefix(r.URL.Path, "/og/")
clean := filepath.Clean(key)
if strings.Contains(clean, "..") {
http.NotFound(w, r)
return
}
path := filepath.Join("storage", "og", clean)
data, err := os.ReadFile(path)
if err != nil {
http.NotFound(w, r)
return
}
w.Header().Set("Content-Type", "image/png")
w.Header().Set("Cache-Control", "public, max-age=3600")
_, _ = w.Write(data)
}
}
// ensureLocalFallbackOG generates the fallback OG image to disk and uploads to R2 if configured.
func ensureLocalFallbackOG(cfg *config.Config) {
path := "storage/og/social/default.png"
data, err := og.GenerateFallbackImage()
if err != nil {
return
}
// Always write locally
_ = os.MkdirAll("storage/og/social", 0o755)
_ = os.WriteFile(path, data, 0o644)
// Upload to R2 CDN if configured
uploader := og.NewUploader(cfg.R2)
if uploader.IsR2() {
_ = uploader.Upload(context.Background(), "social/default.png", data)
}
}
// Query helpers
type indexStats struct {
PluginInstalls int64
ThemeInstalls int64
RootsDownloads int64
UpdatedAt string
}
func queryIndexStats(ctx context.Context, db *sql.DB) indexStats {
var s indexStats
_ = db.QueryRowContext(ctx, "SELECT plugin_installs, theme_installs, COALESCE(updated_at,'') FROM package_stats WHERE id = 1").Scan(&s.PluginInstalls, &s.ThemeInstalls, &s.UpdatedAt)
return s
}
// collapseSlug strips hyphens, underscores, and spaces to produce a
// compact form suitable for LIKE-matching against similarly collapsed names.
func collapseSlug(s string) string {
s = strings.ToLower(s)
s = strings.ReplaceAll(s, "-", "")
s = strings.ReplaceAll(s, "_", "")
s = strings.ReplaceAll(s, " ", "")
return s
}
// ftsQuery converts a user search string into an FTS5 query.
// Each token becomes a prefix search, joined with AND.
// e.g. "woo commerce" -> "woo* AND commerce*"
func ftsQuery(s string) string {
words := strings.Fields(s)
for i, w := range words {
// Escape double quotes to prevent FTS5 syntax injection
w = strings.ReplaceAll(w, `"`, `""`)
words[i] = `"` + w + `"` + "*"
}
return strings.Join(words, " AND ")
}
func queryPackages(ctx context.Context, db *sql.DB, f publicFilters, page, limit int) ([]packageRow, int, error) {
where := "is_active = 1"
args := []any{}
if q := ftsQuery(f.Search); q != "" {
where += " AND (id IN (SELECT rowid FROM packages_fts WHERE packages_fts MATCH ?) OR REPLACE(name, '-', '') LIKE ?)"
args = append(args, q, "%"+collapseSlug(f.Search)+"%")
}
if f.Type != "" {
where += " AND type = ?"
args = append(args, f.Type)
}
orderBy := "wp_packages_installs_total DESC"
switch f.Sort {
case "active_installs":
orderBy = "active_installs DESC"
case "updated":
orderBy = "last_committed DESC NULLS LAST"
case "name":
orderBy = "name ASC"
}
var total int
if f.Search == "" && f.Type == "" {
_ = db.QueryRowContext(ctx, "SELECT active_plugins + active_themes FROM package_stats WHERE id = 1").Scan(&total)
} else {
countQ := "SELECT COUNT(*) FROM packages WHERE " + where
if err := db.QueryRowContext(ctx, countQ, args...).Scan(&total); err != nil {
return nil, 0, err
}
}
offset := (page - 1) * limit
q := fmt.Sprintf(`SELECT type, name, COALESCE(display_name,''), COALESCE(description,''),
COALESCE(current_version,''), downloads, active_installs, wp_packages_installs_total
FROM packages WHERE %s ORDER BY %s LIMIT ? OFFSET ?`, where, orderBy)
rows, err := db.QueryContext(ctx, q, append(args, limit, offset)...)
if err != nil {
return nil, 0, err
}
defer func() { _ = rows.Close() }()
var pkgs []packageRow
for rows.Next() {
var p packageRow
if err := rows.Scan(&p.Type, &p.Name, &p.DisplayName, &p.Description, &p.CurrentVersion, &p.Downloads, &p.ActiveInstalls, &p.WpPackagesInstallsTotal); err != nil {
return nil, 0, fmt.Errorf("scanning package row: %w", err)
}
pkgs = append(pkgs, p)
}
return pkgs, total, rows.Err()
}
type packageDetail struct {
packageRow
WporgVersion string
UpdatedAt string
OGImageGeneratedAt *string
}
func packageExistsInactive(ctx context.Context, db *sql.DB, pkgType, name string) bool {
var n int
err := db.QueryRowContext(ctx,
`SELECT 1 FROM packages WHERE type = ? AND name = ? AND is_active = 0`, pkgType, name).Scan(&n)
return err == nil
}
func queryPackageDetail(ctx context.Context, db *sql.DB, pkgType, name string) (*packageDetail, error) {
var p packageDetail
err := db.QueryRowContext(ctx, `SELECT id, type, name, COALESCE(display_name,''), COALESCE(description,''),
COALESCE(author,''), COALESCE(homepage,''), COALESCE(current_version,''),
downloads, active_installs, wp_packages_installs_total,
COALESCE(wporg_version,''), COALESCE(updated_at,''), og_image_generated_at
FROM packages WHERE type = ? AND name = ? AND is_active = 1`, pkgType, name,
).Scan(&p.ID, &p.Type, &p.Name, &p.DisplayName, &p.Description, &p.Author, &p.Homepage,
&p.CurrentVersion, &p.Downloads, &p.ActiveInstalls, &p.WpPackagesInstallsTotal,
&p.WporgVersion, &p.UpdatedAt, &p.OGImageGeneratedAt)
if err != nil {
return nil, err
}
return &p, nil
}
func versionIsTagged(versions []versionRow, cv string) bool {
for _, v := range versions {
if v.Version == cv {
return true
}
}
return false
}
func hasTaggedVersion(versions []versionRow) bool {
for _, v := range versions {
if v.Version != "dev-trunk" {
return true
}
}
return false
}
func queryDashboardStats(ctx context.Context, db *sql.DB) map[string]any {
stats := map[string]any{
"Stats": struct {
TotalPackages int64
ActivePlugins int64
ActiveThemes int64
TotalInstalls int64
PluginInstalls int64
ThemeInstalls int64
Installs30d int64
CurrentBuild string
StatsUpdatedAt string
}{},
}
var s struct {
TotalPackages int64
ActivePlugins int64
ActiveThemes int64
TotalInstalls int64
PluginInstalls int64
ThemeInstalls int64
Installs30d int64
CurrentBuild string
StatsUpdatedAt string
}
_ = db.QueryRowContext(ctx, `SELECT active_plugins, active_themes, active_plugins + active_themes,
plugin_installs + theme_installs, plugin_installs, theme_installs, installs_30d, COALESCE(updated_at,'') FROM package_stats WHERE id = 1`).Scan(
&s.ActivePlugins, &s.ActiveThemes, &s.TotalPackages, &s.TotalInstalls, &s.PluginInstalls, &s.ThemeInstalls, &s.Installs30d, &s.StatsUpdatedAt)
stats["Stats"] = s
return stats
}
func handleStatus(a *app.App, tmpl *templateSet) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
dashStats := queryDashboardStats(ctx, a.DB)
var lastSyncedAt string
_ = a.DB.QueryRowContext(ctx, `SELECT COALESCE(MAX(last_synced_at), '') FROM packages WHERE is_active = 1`).Scan(&lastSyncedAt)
data := map[string]any{
"Title": "Status",
"Stats": dashStats["Stats"],
"LastSyncedAt": lastSyncedAt,
"SourceURL": "https://wp-packages.org",
"AppURL": a.Config.AppURL,
}
render(w, r, tmpl.status, "layout", data)
}
}