From 86cbb43965d6ec0d71093c53a0589189dd07e023 Mon Sep 17 00:00:00 2001 From: tegwick Date: Thu, 8 Oct 2026 20:55:32 +0200 Subject: [PATCH] FT-WP-0001-T14: upload post images instead of linking them The adapter fetches visual_url from an allowlisted host (https, image types, 10 MB cap, optional read token) and uploads it as multipart sendPhoto, so the forge can close without breaking post images. Co-Authored-By: Claude Opus 5.5 Assistant: claude-code Assistant-Model: opus Assistant-Process: 5457@bnt-lap001 Assistant-Session: 792bf84a-3783-4f5d-a811-cded61dd2012 --- cmd/adapter/main.go | 13 +- internal/adapter/adapter.go | 21 +++ internal/adapter/adapter_test.go | 13 ++ internal/adapter/telegram.go | 54 +++++- internal/adapter/visual.go | 101 +++++++++++ internal/adapter/visual_test.go | 165 ++++++++++++++++++ scripts/send-post.sh | 4 + ...0001-telegram-identity-and-hall-channel.md | 13 +- 8 files changed, 379 insertions(+), 5 deletions(-) create mode 100644 internal/adapter/visual.go create mode 100644 internal/adapter/visual_test.go diff --git a/cmd/adapter/main.go b/cmd/adapter/main.go index c7353e1..e687772 100644 --- a/cmd/adapter/main.go +++ b/cmd/adapter/main.go @@ -13,6 +13,7 @@ import ( "net/http" "os" "strconv" + "strings" "github.com/tegwick/fluid-telegram/internal/adapter" ) @@ -51,8 +52,18 @@ func run() error { AttachVisual: attachVisual, HallBaseURL: hallBaseURL, } + // UPLOAD_VISUALS=true: fetch images from VISUAL_HOSTS (default the forge) + // and upload them, optionally with FORGE_READ_TOKEN, so Telegram never needs + // a public image URL (FT-WP-0001-T14). + if os.Getenv("UPLOAD_VISUALS") == "true" { + hosts := strings.Split(os.Getenv("VISUAL_HOSTS"), ",") + if strings.TrimSpace(os.Getenv("VISUAL_HOSTS")) == "" { + hosts = []string{"forgejo.coulomb.social"} + } + cfg.Visuals = adapter.NewVisualFetcher(hosts, os.Getenv("FORGE_READ_TOKEN")) + } a := adapter.New(cfg, adapter.NewBotAPISender(token), adapter.NewMemStore(), adapter.NewHTTPTelemetry(controlURL)) - log.Printf("adapter listening on %s (chat_id=%d, attach_visual=%v)", addr, chatID, attachVisual) + log.Printf("adapter listening on %s (chat_id=%d, attach_visual=%v, upload_visuals=%v)", addr, chatID, attachVisual, cfg.Visuals != nil) return http.ListenAndServe(addr, a.Handler()) } diff --git a/internal/adapter/adapter.go b/internal/adapter/adapter.go index 8287707..41803e8 100644 --- a/internal/adapter/adapter.go +++ b/internal/adapter/adapter.go @@ -44,8 +44,17 @@ type Config struct { ChatID int64 AttachVisual bool // R-2 (true) attaches a portrait; R-3 (false) is text-only HallBaseURL string + // Visuals, when set, makes the adapter fetch each image itself and upload + // it, so the image's host need not be public. Nil keeps the URL hand-off. + Visuals *VisualFetcher } +// visualError is an image the adapter could not fetch or would not upload. +// It is the request's problem, not Telegram's, so it maps to 400. +type visualError struct{ err error } + +func (e *visualError) Error() string { return e.err.Error() } + // Adapter implements the three R-1 operations. It never writes to Telegram // beyond sending to Config.ChatID, and it holds no MTProto session: see // docs/adapter-contract.md "What the adapter must not do". @@ -217,6 +226,13 @@ func (a *Adapter) handlePublish(w http.ResponseWriter, r *http.Request) { func (a *Adapter) sendNew(ctx context.Context, rendered string, req postRequest) (int, error) { if a.cfg.AttachVisual && req.VisualURL != "" { + if a.cfg.Visuals != nil { + photo, err := a.cfg.Visuals.Fetch(ctx, req.VisualURL) + if err != nil { + return 0, &visualError{err} + } + return a.sndr.UploadPhoto(ctx, a.cfg.ChatID, photo, rendered) + } return a.sndr.SendPhoto(ctx, a.cfg.ChatID, req.VisualURL, rendered) } return a.sndr.SendMessage(ctx, a.cfg.ChatID, rendered) @@ -326,6 +342,11 @@ func writeError(w http.ResponseWriter, status int, msg string) { // limit or any other backend failure becomes 502, never retried automatically // for a call that may have already succeeded. func writeTelegramErr(w http.ResponseWriter, err error) { + var ve *visualError + if errors.As(err, &ve) { + writeError(w, http.StatusBadRequest, "visual_url could not be used: "+ve.Error()) + return + } var rle *RateLimitError if errors.As(err, &rle) { w.Header().Set("Retry-After", fmt.Sprintf("%d", rle.RetryAfterSeconds)) diff --git a/internal/adapter/adapter_test.go b/internal/adapter/adapter_test.go index e8c60cb..220c526 100644 --- a/internal/adapter/adapter_test.go +++ b/internal/adapter/adapter_test.go @@ -15,9 +15,22 @@ type fakeSender struct { sent []string captions []string edits []string + uploads []Photo failNext error } +func (f *fakeSender) UploadPhoto(_ context.Context, _ int64, photo Photo, captionHTML string) (int, error) { + if f.failNext != nil { + err := f.failNext + f.failNext = nil + return 0, err + } + f.nextID++ + f.uploads = append(f.uploads, photo) + f.captions = append(f.captions, captionHTML) + return f.nextID, nil +} + func (f *fakeSender) SendMessage(_ context.Context, _ int64, html string) (int, error) { if f.failNext != nil { err := f.failNext diff --git a/internal/adapter/telegram.go b/internal/adapter/telegram.go index 868392d..866feab 100644 --- a/internal/adapter/telegram.go +++ b/internal/adapter/telegram.go @@ -1,10 +1,13 @@ package adapter import ( + "bytes" "context" "encoding/json" "fmt" + "mime/multipart" "net/http" + "net/textproto" "net/url" "time" ) @@ -17,6 +20,7 @@ import ( type Sender interface { SendMessage(ctx context.Context, chatID int64, html string) (messageID int, err error) SendPhoto(ctx context.Context, chatID int64, photoURL, captionHTML string) (messageID int, err error) + UploadPhoto(ctx context.Context, chatID int64, photo Photo, captionHTML string) (messageID int, err error) EditMessageText(ctx context.Context, chatID int64, messageID int, html string) error EditMessageCaption(ctx context.Context, chatID int64, messageID int, captionHTML string) error } @@ -62,14 +66,22 @@ type apiResult struct { } `json:"parameters"` } +func (b *BotAPISender) endpoint(method string) string { + return fmt.Sprintf("https://api.telegram.org/bot%s/%s", b.token, method) +} + func (b *BotAPISender) call(ctx context.Context, method string, form url.Values) (int, error) { - endpoint := fmt.Sprintf("https://api.telegram.org/bot%s/%s", b.token, method) - req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, nil) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, b.endpoint(method), nil) if err != nil { - return 0, err + return 0, fmt.Errorf("telegram %s: request could not be built", method) } req.URL.RawQuery = form.Encode() + return b.do(method, req) +} +// do sends a prepared request and maps the Bot API reply. Errors never carry +// the request URL, which holds the token. +func (b *BotAPISender) do(method string, req *http.Request) (int, error) { resp, err := b.hc.Do(req) if err != nil { // Never echo the token: err from the http package can embed the @@ -109,6 +121,42 @@ func (b *BotAPISender) SendPhoto(ctx context.Context, chatID int64, photoURL, ca return b.call(ctx, "sendPhoto", form) } +// UploadPhoto sends the image bytes as multipart/form-data, so Telegram never +// needs to reach the image's host. +func (b *BotAPISender) UploadPhoto(ctx context.Context, chatID int64, photo Photo, captionHTML string) (int, error) { + var body bytes.Buffer + mw := multipart.NewWriter(&body) + fields := [][2]string{ + {"chat_id", fmt.Sprintf("%d", chatID)}, + {"caption", captionHTML}, + {"parse_mode", "HTML"}, + } + for _, f := range fields { + if err := mw.WriteField(f[0], f[1]); err != nil { + return 0, fmt.Errorf("telegram sendPhoto: form could not be built") + } + } + h := make(textproto.MIMEHeader) + h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="photo"; filename=%q`, photo.Filename)) + h.Set("Content-Type", photo.ContentType) + part, err := mw.CreatePart(h) + if err != nil { + return 0, fmt.Errorf("telegram sendPhoto: form could not be built") + } + if _, err := part.Write(photo.Data); err != nil { + return 0, fmt.Errorf("telegram sendPhoto: form could not be built") + } + if err := mw.Close(); err != nil { + return 0, fmt.Errorf("telegram sendPhoto: form could not be built") + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, b.endpoint("sendPhoto"), &body) + if err != nil { + return 0, fmt.Errorf("telegram sendPhoto: request could not be built") + } + req.Header.Set("Content-Type", mw.FormDataContentType()) + return b.do("sendPhoto", req) +} + func (b *BotAPISender) EditMessageText(ctx context.Context, chatID int64, messageID int, html string) error { form := url.Values{} form.Set("chat_id", fmt.Sprintf("%d", chatID)) diff --git a/internal/adapter/visual.go b/internal/adapter/visual.go new file mode 100644 index 0000000..7e9ace2 --- /dev/null +++ b/internal/adapter/visual.go @@ -0,0 +1,101 @@ +package adapter + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "path" + "strings" + "time" +) + +// MaxPhotoBytes is Telegram's upload limit for sendPhoto. A larger image is +// refused, not resized: the composer chose it and the reviewer signed it off. +const MaxPhotoBytes = 10 << 20 + +// Photo is an image the adapter fetched itself, ready to upload. Uploading +// rather than handing Telegram a URL means the image's host does not have to +// be public (ORGREF-WP-0002, decision point 4). +type Photo struct { + Data []byte + Filename string + ContentType string +} + +var photoTypes = map[string]bool{"image/jpeg": true, "image/png": true, "image/webp": true} + +// VisualFetcher downloads a post's image from an allowlisted host, optionally +// with a read token. The token is sent only to allowlisted hosts and never +// appears in an error. +type VisualFetcher struct { + hosts map[string]bool + token string + hc *http.Client +} + +// NewVisualFetcher allows fetches from the given hosts only. token may be +// empty while the host still serves the image anonymously. +func NewVisualFetcher(hosts []string, token string) *VisualFetcher { + f := &VisualFetcher{hosts: map[string]bool{}, token: token} + for _, h := range hosts { + if h = strings.TrimSpace(strings.ToLower(h)); h != "" { + f.hosts[h] = true + } + } + f.hc = &http.Client{ + Timeout: 30 * time.Second, + CheckRedirect: func(req *http.Request, _ []*http.Request) error { + if !f.allowed(req.URL) { + return errors.New("redirect to a host that is not allowlisted") + } + return nil + }, + } + return f +} + +func (f *VisualFetcher) allowed(u *url.URL) bool { + return u.Scheme == "https" && f.hosts[strings.ToLower(u.Hostname())] +} + +// Fetch downloads rawURL and checks that it is an image Telegram accepts. +func (f *VisualFetcher) Fetch(ctx context.Context, rawURL string) (Photo, error) { + u, err := url.Parse(rawURL) + if err != nil || !f.allowed(u) { + return Photo{}, fmt.Errorf("visual_url is not an https URL on an allowlisted host") + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return Photo{}, fmt.Errorf("visual_url: %w", err) + } + if f.token != "" { + req.Header.Set("Authorization", "token "+f.token) + } + resp, err := f.hc.Do(req) + if err != nil { + return Photo{}, errors.New("visual fetch: network failure") + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return Photo{}, fmt.Errorf("visual fetch: status %d", resp.StatusCode) + } + ct := strings.ToLower(strings.TrimSpace(strings.SplitN(resp.Header.Get("Content-Type"), ";", 2)[0])) + if !photoTypes[ct] { + return Photo{}, fmt.Errorf("visual fetch: content type %q is not jpeg, png or webp", ct) + } + data, err := io.ReadAll(io.LimitReader(resp.Body, MaxPhotoBytes+1)) + if err != nil { + return Photo{}, errors.New("visual fetch: read failure") + } + if len(data) > MaxPhotoBytes { + return Photo{}, fmt.Errorf("visual fetch: image is over Telegram's %d byte photo limit", MaxPhotoBytes) + } + name := path.Base(u.Path) + if name == "" || name == "/" || name == "." { + name = "visual" + } + return Photo{Data: data, Filename: name, ContentType: ct}, nil +} diff --git a/internal/adapter/visual_test.go b/internal/adapter/visual_test.go new file mode 100644 index 0000000..7b6e611 --- /dev/null +++ b/internal/adapter/visual_test.go @@ -0,0 +1,165 @@ +package adapter + +import ( + "bytes" + "context" + "io" + "mime" + "mime/multipart" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" +) + +var pngBytes = []byte("\x89PNG\r\n\x1a\nfake-image-bytes") + +// imageServer serves one image over TLS and records the Authorization header. +func imageServer(t *testing.T, contentType string, body []byte) (*httptest.Server, *string) { + t.Helper() + var auth string + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + auth = r.Header.Get("Authorization") + w.Header().Set("Content-Type", contentType) + _, _ = w.Write(body) + })) + t.Cleanup(srv.Close) + return srv, &auth +} + +func fetcherFor(srv *httptest.Server, token string) *VisualFetcher { + u, _ := url.Parse(srv.URL) + f := NewVisualFetcher([]string{u.Hostname()}, token) + client := srv.Client() + client.CheckRedirect = f.hc.CheckRedirect + f.hc = client + return f +} + +func visualReq(t *testing.T, visualURL string) postRequest { + t.Helper() + req := validReq("p-visual") + req.VisualURL = visualURL + return req +} + +func TestPublishUploadsFetchedImage(t *testing.T) { + srv, auth := imageServer(t, "image/png", pngBytes) + snd := &fakeSender{} + a := New(Config{ChatID: 1, AttachVisual: true, Visuals: fetcherFor(srv, "read-token-123")}, snd, NewMemStore(), nil) + w := post(t, a.Handler(), visualReq(t, srv.URL+"/coulomb/x/raw/branch/main/assets/a.png")) + if w.Code != http.StatusCreated { + t.Fatalf("status = %d, body = %s", w.Code, w.Body.String()) + } + if len(snd.uploads) != 1 || !bytes.Equal(snd.uploads[0].Data, pngBytes) { + t.Fatalf("expected the fetched bytes uploaded, got %d uploads", len(snd.uploads)) + } + if snd.uploads[0].Filename != "a.png" || snd.uploads[0].ContentType != "image/png" { + t.Fatalf("upload = %+v", snd.uploads[0]) + } + if *auth != "token read-token-123" { + t.Fatalf("authorization header = %q", *auth) + } +} + +func TestPublishRefusesUnusableVisuals(t *testing.T) { + big := append([]byte{}, pngBytes...) + big = append(big, bytes.Repeat([]byte{0}, MaxPhotoBytes)...) + cases := []struct { + name, ct string + body []byte + other bool + }{ + {"host not allowlisted", "image/png", pngBytes, true}, + {"not an image", "text/html", []byte(""), false}, + {"over the photo limit", "image/png", big, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + srv, _ := imageServer(t, tc.ct, tc.body) + f := fetcherFor(srv, "read-token-123") + if tc.other { + f = NewVisualFetcher([]string{"forgejo.example"}, "read-token-123") + } + snd := &fakeSender{} + a := New(Config{ChatID: 1, AttachVisual: true, Visuals: f}, snd, NewMemStore(), nil) + w := post(t, a.Handler(), visualReq(t, srv.URL+"/a.png")) + if w.Code != http.StatusBadRequest { + t.Fatalf("status = %d, body = %s", w.Code, w.Body.String()) + } + if len(snd.uploads)+len(snd.sent)+len(snd.captions) != 0 { + t.Fatal("something was sent to Telegram") + } + if strings.Contains(w.Body.String(), "read-token-123") { + t.Fatal("the read token appeared in the response") + } + }) + } +} + +func TestFetchRefusesPlainHTTP(t *testing.T) { + f := NewVisualFetcher([]string{"forgejo.coulomb.social"}, "") + if _, err := f.Fetch(context.Background(), "http://forgejo.coulomb.social/a.png"); err == nil { + t.Fatal("fetched over plain http") + } +} + +type captureTransport struct { + req *http.Request + body []byte +} + +func (c *captureTransport) RoundTrip(r *http.Request) (*http.Response, error) { + c.req = r + c.body, _ = io.ReadAll(r.Body) + return &http.Response{StatusCode: 200, Header: http.Header{"Content-Type": {"application/json"}}, + Body: io.NopCloser(strings.NewReader(`{"ok":true,"result":{"message_id":9}}`))}, nil +} + +func TestBotAPIUploadPhotoIsMultipart(t *testing.T) { + ct := &captureTransport{} + b := NewBotAPISender("123:secret-bot-token") + b.hc = &http.Client{Transport: ct} + id, err := b.UploadPhoto(context.Background(), -1004427430639, + Photo{Data: pngBytes, Filename: "a.png", ContentType: "image/png"}, "caption x") + if err != nil || id != 9 { + t.Fatalf("id=%d err=%v", id, err) + } + if ct.req.URL.Path != "/bot123:secret-bot-token/sendPhoto" { + t.Fatalf("path = %s", ct.req.URL.Path) + } + _, params, err := mime.ParseMediaType(ct.req.Header.Get("Content-Type")) + if err != nil { + t.Fatal(err) + } + mr := multipart.NewReader(bytes.NewReader(ct.body), params["boundary"]) + got := map[string][]byte{} + for { + p, err := mr.NextPart() + if err != nil { + break + } + data, _ := io.ReadAll(p) + got[p.FormName()] = data + } + if string(got["chat_id"]) != "-1004427430639" || string(got["parse_mode"]) != "HTML" || + string(got["caption"]) != "caption x" || !bytes.Equal(got["photo"], pngBytes) { + t.Fatalf("multipart fields = %v", got) + } +} + +type failTransport struct{} + +func (failTransport) RoundTrip(*http.Request) (*http.Response, error) { + return nil, io.ErrUnexpectedEOF +} + +func TestBotAPIUploadErrorHidesToken(t *testing.T) { + b := NewBotAPISender("123:secret-bot-token") + b.hc = &http.Client{Transport: failTransport{}} + _, err := b.UploadPhoto(context.Background(), 1, Photo{Data: pngBytes, Filename: "a.png", ContentType: "image/png"}, "c") + if err == nil || strings.Contains(err.Error(), "secret-bot-token") { + t.Fatalf("err = %v", err) + } +} diff --git a/scripts/send-post.sh b/scripts/send-post.sh index 7ee2fbc..fd1f122 100755 --- a/scripts/send-post.sh +++ b/scripts/send-post.sh @@ -83,7 +83,11 @@ token=$(bao read -format=json \ | python3 -c 'import sys, json; print(json.load(sys.stdin)["data"]["data"]["token"])') \ || { rm -rf "$tmp"; echo "could not read the bot token from OpenBao (logged in? read on platform/data/workloads/infotech/fluid-telegram/hall-of-helix/*?)" >&2; exit 2; } +# Images are fetched by the adapter and uploaded (FT-WP-0001-T14), so the +# forge need not serve them publicly. FORGE_READ_TOKEN, when set, is a +# read-only forge identity for closed repositories. TELEGRAM_BOT_TOKEN=$token TELEGRAM_CHANNEL_ID=$TELEGRAM_CHANNEL_ID ATTACH_VISUAL=true \ + UPLOAD_VISUALS=true FORGE_READ_TOKEN=${FORGE_READ_TOKEN:-} \ ADAPTER_LISTEN_ADDR=127.0.0.1:$port "$tmp/adapter" & pid=$! unset token diff --git a/workplans/FT-WP-0001-telegram-identity-and-hall-channel.md b/workplans/FT-WP-0001-telegram-identity-and-hall-channel.md index b7fd4a1..d53d397 100644 --- a/workplans/FT-WP-0001-telegram-identity-and-hall-channel.md +++ b/workplans/FT-WP-0001-telegram-identity-and-hall-channel.md @@ -439,7 +439,7 @@ you have used it is how you find out you cut it in the wrong place. ```task id: FT-WP-0001-T14 -status: todo +status: progress priority: high state_hub_task_id: "c822885b-c2a2-57a7-a448-7d1a853a26c0" ``` @@ -462,3 +462,14 @@ replaces the media the same way. No public asset location is improvised for this. A dedicated asset store with creator and trust management comes later, on its own. + +2026-10-08: implemented. `internal/adapter/visual.go` fetches the image from an +allowlisted host: https only, no redirects to other hosts, JPEG/PNG/WebP only, +refused over 10 MB, with an optional `FORGE_READ_TOKEN` sent only to that host. +`BotAPISender.UploadPhoto` sends it as multipart `sendPhoto`. Enabled with +`UPLOAD_VISUALS=true` (on in `scripts/send-post.sh`). Tests cover upload, +refusals, the token never appearing in responses or errors, and the multipart +fields. Dry run fetched the real image. Open: one signed test post to the +private channel (`pr-hall-of-helix/posts/261008T205518-test-image-upload.yaml`). +Editing an existing photo post still edits only the caption; replacing the +image would be a separate change.