tenseleyflow/shithub / 5197fee

+- [seo.md](./seo.md) — crawler endpoints, metadata, sitemap, and

+  public positioning.

+# SEO and crawler surfaces

+

+shithub exposes a small, honest crawler surface for the hosted site and

+self-hosted instances:

+

+- `GET /robots.txt` allows public pages and points crawlers at

+  `/sitemap.xml`.

+- `GET /sitemap.xml` lists stable public marketing/discovery pages:

+  `/`, `/about`, `/explore`, and `/trending`.

+- The shared HTML layout emits a page description, canonical URL,

+  Open Graph metadata, Twitter card metadata, and trusted JSON-LD when

+  handlers provide those fields. Missing fields are optional for typed

+  page-data structs.

+- `/about` is the durable positioning page. It follows the README,

+  SECURITY, CONTRIBUTING, and CODE_OF_CONDUCT posture: GitHub is good

+  software, shithub exists because users should be able to host code

+  without AI training concerns, and the community is AGPL, security-aware,

+  and civil.

+

+Use `auth.base_url` as the public origin in production. It drives the

+canonical links generated by crawler endpoints. If it is empty in tests

+or local dev, handlers fall back to the request host.

+

+Do not add generated search result pages, settings pages, API routes,

+admin pages, or smart-HTTP `.git` endpoints to the sitemap.

+	// BaseURL is the public scheme+host for canonical crawler URLs

+	// (for example https://shithub.sh). Empty falls back to the request

+	// host, which keeps tests and local dev working.

+	BaseURL string

+		crawlers := crawlerHandler{baseURL: deps.BaseURL}

+		r.Get("/robots.txt", crawlers.serveRobots)

+		r.Get("/sitemap.xml", crawlers.serveSitemap)

+		marketing := marketingHandler{render: rr, baseURL: deps.BaseURL, logger: deps.Logger}

+		r.Get("/", helloHandler{render: rr, logoSVG: deps.LogoSVG, baseURL: deps.BaseURL, logger: deps.Logger}.ServeHTTP)

+		r.Get("/about", marketing.serveAbout)

+			wantBodyAny: []string{"shithub", "GitHub. Open source. Without Copilot.", "Sprint 00", `<meta name="description"`, `<link rel="canonical"`},

+		{

+			name:        "about page",

+			path:        "/about",

+			wantStatus:  http.StatusOK,

+			wantBodyAny: []string{"No hard feelings to GitHub", "AI training on my code", `<meta name="description"`},

+			wantHeader:  map[string]string{"Content-Type": "text/html; charset=utf-8"},

+		},

+		{

+			name:        "robots",

+			path:        "/robots.txt",

+			wantStatus:  http.StatusOK,

+			wantBodyAny: []string{"User-agent: *", "Allow: /", "Disallow: /admin", "Sitemap: http://example.com/sitemap.xml"},

+			wantHeader:  map[string]string{"Content-Type": "text/plain; charset=utf-8"},

+		},

+		{

+			name:        "sitemap",

+			path:        "/sitemap.xml",

+			wantStatus:  http.StatusOK,

+			wantBodyAny: []string{`<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">`, "<loc>http://example.com/</loc>", "<loc>http://example.com/about</loc>"},

+			wantHeader:  map[string]string{"Content-Type": "application/xml; charset=utf-8"},

+		},

+	baseURL string

+	// page-data structs must populate them explicitly - the renderer

+	// SEO/social fields are read by optional layout helpers, so typed

+	// page-data structs may provide only the fields they actually need.

+	MetaDescription string

+	CanonicalURL    string

+	OGTitle         string

+	OGDescription   string

+	OGImage         string

+	OGType          string

+	StructuredData  template.JS

+		Title:           "Welcome",

+		Version:         version.Version,

+		Commit:          version.Commit,

+		BuiltAt:         version.BuiltAt,

+		LogoSVG:         template.HTML(h.logoSVG), // #nosec G203 - embedded server-owned asset

+		Viewer:          viewer,

+		CSRFToken:       middleware.CSRFTokenForRequest(r),

+		MetaDescription: defaultMetaDescription,

+		CanonicalURL:    canonicalURL(h.baseURL, r, "/"),

+		OGTitle:         "shithub: GitHub-style git hosting without Copilot",

+		OGDescription:   "A self-hostable, AGPL GitHub alternative with familiar repositories, pull requests, issues, organizations, code search, and Actions-style CI.",

+		OGType:          "website",

+		StructuredData:  organizationStructuredData(publicBaseURL(h.baseURL, r)),

+// SPDX-License-Identifier: AGPL-3.0-or-later

+

+package handlers

+

+import (

+	"encoding/json"

+	"encoding/xml"

+	"fmt"

+	"html/template"

+	"log/slog"

+	"net/http"

+	"net/url"

+	"strings"

+

+	"github.com/tenseleyFlow/shithub/internal/web/middleware"

+	"github.com/tenseleyFlow/shithub/internal/web/render"

+)

+

+const defaultMetaDescription = "shithub is an AGPL self-hosted GitHub alternative: Git repositories, pull requests, issues, Actions-style CI, organizations, and code search without Copilot."

+

+type marketingHandler struct {

+	render  *render.Renderer

+	baseURL string

+	logger  *slog.Logger

+}

+

+type crawlerHandler struct {

+	baseURL string

+}

+

+var sitemapPaths = []string{

+	"/",

+	"/about",

+	"/explore",

+	"/trending",

+}

+

+func (h marketingHandler) serveAbout(w http.ResponseWriter, r *http.Request) {

+	data := map[string]any{

+		"Title":             "GitHub alternative for self-hosted Git teams",

+		"MetaDescription":   "shithub is a self-hosted git forge and GitHub alternative for teams that want familiar pull requests, issues, Actions-style CI, and open-source control without Copilot.",

+		"CanonicalURL":      canonicalURL(h.baseURL, r, "/about"),

+		"OGTitle":           "shithub: self-hosted GitHub alternative",

+		"OGDescription":     "An AGPL git forge with GitHub-style repositories, pull requests, issues, organizations, code search, and Actions-style CI without Copilot.",

+		"StructuredData":    organizationStructuredData(publicBaseURL(h.baseURL, r)),

+		"GlobalSearchQuery": "",

+		"Viewer":            middleware.CurrentUserFromContext(r.Context()),

+		"CSRFToken":         middleware.CSRFTokenForRequest(r),

+		"Repo":              nil,

+		"Org":               nil,

+	}

+	if err := h.render.RenderPage(w, r, "about", data); err != nil {

+		h.logger.Error("render about", "error", err)

+		http.Error(w, "internal server error", http.StatusInternalServerError)

+	}

+}

+

+func (h crawlerHandler) serveRobots(w http.ResponseWriter, r *http.Request) {

+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")

+	w.Header().Set("Cache-Control", "public, max-age=300, must-revalidate")

+

+	sitemap := canonicalURL(h.baseURL, r, "/sitemap.xml")

+	fmt.Fprintln(w, "User-agent: *")

+	fmt.Fprintln(w, "Allow: /")

+	fmt.Fprintln(w, "Disallow: /admin")

+	fmt.Fprintln(w, "Disallow: /api/")

+	fmt.Fprintln(w, "Disallow: /internal/")

+	fmt.Fprintln(w, "Disallow: /notifications")

+	fmt.Fprintln(w, "Disallow: /search")

+	fmt.Fprintln(w, "Disallow: /settings")

+	fmt.Fprintln(w, "Disallow: /*.git/")

+	if sitemap != "" {

+		fmt.Fprintln(w, "Sitemap: "+sitemap) // #nosec G705 -- text/plain robots body; request-host fallback rejects control characters.

+	}

+}

+

+func (h crawlerHandler) serveSitemap(w http.ResponseWriter, r *http.Request) {

+	w.Header().Set("Content-Type", "application/xml; charset=utf-8")

+	w.Header().Set("Cache-Control", "public, max-age=300, must-revalidate")

+

+	fmt.Fprintln(w, xml.Header+`<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">`)

+	for _, p := range sitemapPaths {

+		loc := canonicalURL(h.baseURL, r, p)

+		if loc == "" {

+			continue

+		}

+		fmt.Fprint(w, "  <url><loc>")

+		_ = xml.EscapeText(w, []byte(loc))

+		fmt.Fprintln(w, "</loc></url>")

+	}

+	fmt.Fprintln(w, "</urlset>")

+}

+

+func canonicalURL(configuredBase string, r *http.Request, p string) string {

+	base := publicBaseURL(configuredBase, r)

+	if base == "" {

+		return ""

+	}

+	if p == "" {

+		p = "/"

+	}

+	if !strings.HasPrefix(p, "/") {

+		p = "/" + p

+	}

+	return base + p

+}

+

+func publicBaseURL(configuredBase string, r *http.Request) string {

+	if base := strings.TrimRight(strings.TrimSpace(configuredBase), "/"); base != "" {

+		return base

+	}

+	if r == nil || r.Host == "" {

+		return ""

+	}

+	host := strings.TrimSpace(r.Host)

+	if host == "" || strings.ContainsAny(host, "\r\n\t /\\") {

+		return ""

+	}

+	scheme := "http"

+	if r.TLS != nil {

+		scheme = "https"

+	}

+	return (&url.URL{Scheme: scheme, Host: host}).String()

+}

+

+func organizationStructuredData(baseURL string) template.JS {

+	if baseURL == "" {

+		return ""

+	}

+	payload := map[string]any{

+		"@context":    "https://schema.org",

+		"@type":       "Organization",

+		"name":        "shithub",

+		"url":         baseURL,

+		"logo":        baseURL + "/static/logo/shithub-mark.svg",

+		"description": defaultMetaDescription,

+		"sameAs": []string{

+			"https://github.com/tenseleyFlow/shithub",

+		},

+	}

+	raw, _ := json.Marshal(payload)

+	return template.JS(raw) // #nosec G203 -- payload is marshaled from server-owned constants and URLs.

+}

+// SPDX-License-Identifier: AGPL-3.0-or-later

+

+package handlers

+

+import (

+	"net/http"

+	"net/http/httptest"

+	"testing"

+)

+

+func TestPublicBaseURLRejectsUnsafeRequestHostFallback(t *testing.T) {

+	t.Parallel()

+

+	req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)

+	req.Host = "example.com\r\nSitemap: https://evil.test/sitemap.xml"

+	if got := publicBaseURL("", req); got != "" {

+		t.Fatalf("publicBaseURL accepted unsafe host = %q", got)

+	}

+}

+

+func TestPublicBaseURLPrefersConfiguredBase(t *testing.T) {

+	t.Parallel()

+

+	req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)

+	req.Host = "untrusted.example"

+	if got, want := publicBaseURL("https://shithub.sh/", req), "https://shithub.sh"; got != want {

+		t.Fatalf("publicBaseURL = %q, want %q", got, want)

+	}

+}

+			Data: []byte(`{{ define "layout" }}<!DOCTYPE html><html><head>{{ if stringField . "MetaDescription" }}<meta name="description" content="{{ stringField . "MetaDescription" }}">{{ end }}{{ with stringField . "CanonicalURL" }}<link rel="canonical" href="{{ . }}">{{ end }}<title>{{ .Title }} · shithub</title></head><body>{{ template "page" . }}</body></html>{{ end }}`),

+		"about.html": &fstest.MapFile{

+			Data: []byte(`{{ define "page" }}<main>No hard feelings to GitHub. I just don't want AI training on my code.</main>{{ end }}`),

+		},

+		// stringField reads an optional string-ish field from map or

+		// struct template data. Shared document metadata uses this so

+		// typed page-data structs don't all have to define every optional

+		// SEO/social field.

+		"stringField": dataString,

+		// jsField returns only server-marked template.JS values. It is

+		// intentionally stricter than stringField so JSON-LD can be

+		// emitted from trusted marshaled data without giving templates a

+		// generic raw-JS escape hatch for arbitrary strings.

+		"jsField": dataJS,

+	field, ok := dataField(data, name)

+	if !ok {

+		return false

+	}

+	return truthyValue(field)

+}

+

+func dataString(data any, name string) string {

+	field, ok := dataField(data, name)

+	if !ok {

+		return ""

+	}

+	for field.Kind() == reflect.Pointer || field.Kind() == reflect.Interface {

+		if field.IsNil() {

+			return ""

+		}

+		field = field.Elem()

+	}

+	switch field.Kind() {

+	case reflect.String:

+		return field.String()

+	default:

+		if field.CanInterface() {

+			return fmt.Sprint(field.Interface())

+		}

+		return ""

+	}

+}

+

+func dataJS(data any, name string) template.JS {

+	field, ok := dataField(data, name)

+	if !ok {

+		return ""

+	}

+	if field.CanInterface() {

+		switch v := field.Interface().(type) {

+		case template.JS:

+			return v

+		case *template.JS:

+			if v != nil {

+				return *v

+			}

+		}

+	}

+	for field.Kind() == reflect.Pointer || field.Kind() == reflect.Interface {

+		if field.IsNil() {

+			return ""

+		}

+		field = field.Elem()

+		if field.CanInterface() {

+			if v, ok := field.Interface().(template.JS); ok {

+				return v

+			}

+		}

+	}

+	return ""

+}

+

+func dataField(data any, name string) (reflect.Value, bool) {

+		return reflect.Value{}, false

+			return reflect.Value{}, false

+			return reflect.Value{}, false

+		if !field.IsValid() {

+			return reflect.Value{}, false

+		}

+		return field, true

+			return reflect.Value{}, false

+		return field, true

+		return reflect.Value{}, false

+	"html/template"

+func TestStringFieldHelperSupportsOptionalLayoutMetadata(t *testing.T) {

+	t.Parallel()

+	fsys := fstest.MapFS{

+		"_layout.html": &fstest.MapFile{Data: []byte(

+			`{{ define "layout" }}<html><head>{{ with stringField . "MetaDescription" }}<meta name="description" content="{{ . }}">{{ end }}</head>{{ template "page" . }}</html>{{ end }}`,

+		)},

+		"page.html": &fstest.MapFile{Data: []byte(`{{ define "page" }}body{{ end }}`)},

+	}

+	r, err := New(fsys, Options{})

+	if err != nil {

+		t.Fatalf("New: %v", err)

+	}

+	var buf bytes.Buffer

+	type typedPageData struct {

+		Title string

+	}

+	if err := r.Render(&buf, "page", typedPageData{Title: "typed"}); err != nil {

+		t.Fatalf("render typed data without metadata: %v", err)

+	}

+	if strings.Contains(buf.String(), `name="description"`) {

+		t.Fatalf("absent typed metadata rendered meta tag: %q", buf.String())

+	}

+	buf.Reset()

+	if err := r.Render(&buf, "page", map[string]any{"MetaDescription": "self-hosted git forge"}); err != nil {

+		t.Fatalf("render map data with metadata: %v", err)

+	}

+	if !strings.Contains(buf.String(), `content="self-hosted git forge"`) {

+		t.Fatalf("map metadata did not render: %q", buf.String())

+	}

+}

+

+func TestJSFieldHelperOnlyRendersTrustedJS(t *testing.T) {

+	t.Parallel()

+	fsys := fstest.MapFS{

+		"_layout.html": &fstest.MapFile{Data: []byte(

+			`{{ define "layout" }}<html><head>{{ with jsField . "StructuredData" }}<script type="application/ld+json">{{ . }}</script>{{ end }}</head>{{ template "page" . }}</html>{{ end }}`,

+		)},

+		"page.html": &fstest.MapFile{Data: []byte(`{{ define "page" }}body{{ end }}`)},

+	}

+	r, err := New(fsys, Options{})

+	if err != nil {

+		t.Fatalf("New: %v", err)

+	}

+	var buf bytes.Buffer

+	if err := r.Render(&buf, "page", map[string]any{"StructuredData": `{"unsafe":true}`}); err != nil {

+		t.Fatalf("render map data with plain string metadata: %v", err)

+	}

+	if strings.Contains(buf.String(), "application/ld+json") {

+		t.Fatalf("plain string JSON-LD should not render as trusted JS: %q", buf.String())

+	}

+	buf.Reset()

+	if err := r.Render(&buf, "page", map[string]any{"StructuredData": template.JS(`{"@type":"Organization"}`)}); err != nil {

+		t.Fatalf("render map data with trusted metadata: %v", err)

+	}

+	if !strings.Contains(buf.String(), `{"@type":"Organization"}`) {

+		t.Fatalf("trusted JSON-LD did not render: %q", buf.String())

+	}

+}

+

+		BaseURL:      cfg.Auth.BaseURL,

+/* ========== About page ========== */

+

+.shithub-about {

+  max-width: 920px;

+  margin: 4rem auto;

+  padding: 0 1.5rem 2rem;

+}

+.shithub-about-hero {

+  max-width: 760px;

+  margin-bottom: 2rem;

+}

+.shithub-about-kicker {

+  margin: 0 0 0.75rem;

+  color: var(--accent-fg);

+  font-weight: 600;

+  font-size: 0.9rem;

+}

+.shithub-about h1 {

+  margin: 0 0 1rem;

+  font-size: 2.75rem;

+  line-height: 1.08;

+}

+.shithub-about-hero p,

+.shithub-about-section p {

+  color: var(--fg-muted);

+  font-size: 1.05rem;

+  line-height: 1.65;

+}

+.shithub-about-section {

+  margin-top: 2rem;

+}

+.shithub-about-section h2 {

+  margin: 0 0 0.75rem;

+  font-size: 1.35rem;

+}

+.shithub-about-list {

+  list-style: none;

+  margin: 0;

+  padding: 0;

+  display: grid;

+  grid-template-columns: repeat(auto-fit, minmax(16rem, 1fr));

+  gap: 1rem;

+}

+.shithub-about-list li {

+  border: 1px solid var(--border-default);

+  border-radius: 6px;

+  padding: 1rem;

+  background: var(--canvas-subtle);

+}

+.shithub-about-list strong {

+  display: block;

+  margin-bottom: 0.35rem;

+}

+.shithub-about-list span {

+  display: block;

+  color: var(--fg-muted);

+  font-size: 0.95rem;

+  line-height: 1.5;

+}

+.shithub-about-actions {

+  display: flex;

+  flex-wrap: wrap;

+  gap: 0.75rem;

+  margin-top: 2rem;

+}

+.shithub-about-actions a {

+  display: inline-block;

+  padding: 0.5rem 1rem;

+  border-radius: 6px;

+  text-decoration: none;

+  font-weight: 500;

+  font-size: 0.95rem;

+}

+@media (max-width: 640px) {

+  .shithub-about {

+    margin: 2.5rem auto;

+  }

+  .shithub-about h1 {

+    font-size: 2rem;

+  }

+}

+

+      <a href="https://docs.shithub.sh/">Docs</a>

+      <a href="https://docs.shithub.sh/security.html">Security</a>

+      <a href="/shithub/shithub">Source</a>

+  {{ if stringField . "MetaDescription" }}<meta name="description" content="{{ stringField . "MetaDescription" }}">{{ else }}<meta name="description" content="shithub is an AGPL self-hosted GitHub alternative: Git repositories, pull requests, issues, Actions-style CI, organizations, and code search without Copilot.">{{ end }}

+  {{ with stringField . "CanonicalURL" }}<link rel="canonical" href="{{ . }}">{{ end }}

+  <meta property="og:site_name" content="shithub">

+  {{ if stringField . "OGType" }}<meta property="og:type" content="{{ stringField . "OGType" }}">{{ else }}<meta property="og:type" content="website">{{ end }}

+  {{ if stringField . "OGTitle" }}<meta property="og:title" content="{{ stringField . "OGTitle" }}">{{ else }}<meta property="og:title" content="{{ .Title }} · shithub">{{ end }}

+  {{ if stringField . "OGDescription" }}<meta property="og:description" content="{{ stringField . "OGDescription" }}">{{ else if stringField . "MetaDescription" }}<meta property="og:description" content="{{ stringField . "MetaDescription" }}">{{ end }}

+  {{ with stringField . "CanonicalURL" }}<meta property="og:url" content="{{ . }}">{{ end }}

+  {{ with stringField . "OGImage" }}<meta property="og:image" content="{{ . }}">{{ end }}

+  {{ if stringField . "OGImage" }}<meta name="twitter:card" content="summary_large_image">{{ else }}<meta name="twitter:card" content="summary">{{ end }}

+  {{ if stringField . "OGTitle" }}<meta name="twitter:title" content="{{ stringField . "OGTitle" }}">{{ else }}<meta name="twitter:title" content="{{ .Title }} · shithub">{{ end }}

+  {{ if stringField . "OGDescription" }}<meta name="twitter:description" content="{{ stringField . "OGDescription" }}">{{ else if stringField . "MetaDescription" }}<meta name="twitter:description" content="{{ stringField . "MetaDescription" }}">{{ end }}

+  {{ with jsField . "StructuredData" }}<script type="application/ld+json">{{ . }}</script>{{ end }}

+{{ define "page" -}}

+<section class="shithub-about">

+  <header class="shithub-about-hero">

+    <p class="shithub-about-kicker">Self-hosted GitHub alternative</p>

+    <h1>Git hosting with GitHub muscle memory and no AI training on your code.</h1>

+    <p>

+      shithub is an AGPL git forge for teams who like GitHub's product,

+      workflows, and interface, but want a self-hostable home for source

+      code that is not part of a Copilot training pipeline.

+    </p>

+  </header>

+

+  <section class="shithub-about-section">

+    <h2>No hard feelings</h2>

+    <p>

+      GitHub is good software. The goal here is not to pretend otherwise.

+      The issue is simpler: I just don't want AI training on my code.

+      shithub keeps the parts that work: repositories, pull requests,

+      issues, code review, organizations, search, and Actions-style CI,

+      while making the hosted service and the self-hosted product open

+      under the AGPL.

+    </p>

+  </section>

+

+  <section class="shithub-about-section">

+    <h2>What shithub is trying to be</h2>

+    <ul class="shithub-about-list">

+      <li>

+        <strong>A real forge, not a demo.</strong>

+        <span>Git repositories, HTTPS push and pull, reviews, required checks, webhooks, notifications, teams, scoped tokens, and code search are product requirements, not stretch goals.</span>

+      </li>

+      <li>

+        <strong>Familiar on purpose.</strong>

+        <span>The closer the GitHub-style UI and workflows are, the less expensive it is for a team to move.</span>

+      </li>

+      <li>

+        <strong>Open when hosted.</strong>

+        <span>AGPLv3 means modified network services have to publish the matching source, so hosted shithub instances stay accountable to their users.</span>

+      </li>

+      <li>

+        <strong>Honest about gaps.</strong>

+        <span>The public docs and README call out what is live, what is rough, and what is still in flight.</span>

+      </li>

+    </ul>

+  </section>

+

+  <section class="shithub-about-section">

+    <h2>Community posture</h2>

+    <p>

+      Contributions are welcome under the same AGPL terms, security reports

+      go through coordinated disclosure, and the project follows the

+      Contributor Covenant. Be direct, be civil, and keep the work focused

+      on making an open GitHub alternative that people can actually use.

+    </p>

+  </section>

+

+  <nav class="shithub-about-actions" aria-label="About links">

+    <a class="shithub-landing-cta-primary" href="/signup">Create an account</a>

+    <a class="shithub-landing-cta-secondary" href="https://docs.shithub.sh/self-host/deploy.html">Self-host shithub</a>

+    <a class="shithub-landing-cta-secondary" href="/shithub/shithub">Read the source</a>

+    <a class="shithub-landing-cta-secondary" href="https://docs.shithub.sh/security.html">Security policy</a>

+  </nav>

+</section>

+{{- end }}