diff --git a/v2/ezytel/ezytel.pb.go b/v2/ezytel/ezytel.pb.go index 5f9bce1ca..5a51d3340 100644 --- a/v2/ezytel/ezytel.pb.go +++ b/v2/ezytel/ezytel.pb.go @@ -29,9 +29,14 @@ type ChannelInfoRequest struct { ChannelId string `protobuf:"bytes,1,opt,name=channel_id,json=channelId,proto3" json:"channel_id,omitempty"` // Last post id the client has already read; used to compute the // unread badge (newmsg). 0 means "everything is unread". - LastRead int64 `protobuf:"varint,2,opt,name=last_read,json=lastRead,proto3" json:"last_read,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + LastRead int64 `protobuf:"varint,2,opt,name=last_read,json=lastRead,proto3" json:"last_read,omitempty"` + // By default avatar_path in the response carries a + // "data:image/jpeg;base64,…" URI ready to drop into . + // Set this to true to opt out and receive the legacy + // "cache/.jpg" server-side file path instead. + DisableInlineImages bool `protobuf:"varint,3,opt,name=disable_inline_images,json=disableInlineImages,proto3" json:"disable_inline_images,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ChannelInfoRequest) Reset() { @@ -78,13 +83,22 @@ func (x *ChannelInfoRequest) GetLastRead() int64 { return 0 } +func (x *ChannelInfoRequest) GetDisableInlineImages() bool { + if x != nil { + return x.DisableInlineImages + } + return false +} + type ChannelInfo struct { state protoimpl.MessageState `protogen:"open.v1"` Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` // Plain-text preview of the most recent post, or "فایل" if it is a media post. Description string `protobuf:"bytes,2,opt,name=description,proto3" json:"description,omitempty"` - // Local cache path of the avatar JPEG, relative to the cache dir - // returned by GetCacheDir(). Empty if the avatar could not be fetched. + // By default this is a "data:image/jpeg;base64,…" URI ready for + // . When the request set disable_inline_images=true it is + // the legacy "cache/.jpg" file path relative to GetCacheDir(). + // Empty if the avatar could not be fetched. AvatarPath string `protobuf:"bytes,3,opt,name=avatar_path,json=avatarPath,proto3" json:"avatar_path,omitempty"` // Unix epoch seconds of the most recent post. Date int64 `protobuf:"varint,4,opt,name=date,proto3" json:"date,omitempty"` @@ -193,9 +207,15 @@ type ChannelMessagesRequest struct { ChannelId string `protobuf:"bytes,1,opt,name=channel_id,json=channelId,proto3" json:"channel_id,omitempty"` // 0 = latest page; otherwise the "before" cursor returned by Telegram // (the data-before attribute of messages_more_wrap). - Before int64 `protobuf:"varint,2,opt,name=before,proto3" json:"before,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + Before int64 `protobuf:"varint,2,opt,name=before,proto3" json:"before,omitempty"` + // By default every image URL in the returned html and the + // channel_avatar field is a "data:image/jpeg;base64,…" URI ready + // to render. Set this to true to opt out: html and channel_avatar + // will both carry the legacy "proxy.php?url=" placeholder + // form, requiring per-image ProxyImage calls. + DisableInlineImages bool `protobuf:"varint,3,opt,name=disable_inline_images,json=disableInlineImages,proto3" json:"disable_inline_images,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ChannelMessagesRequest) Reset() { @@ -242,13 +262,25 @@ func (x *ChannelMessagesRequest) GetBefore() int64 { return 0 } +func (x *ChannelMessagesRequest) GetDisableInlineImages() bool { + if x != nil { + return x.DisableInlineImages + } + return false +} + type ChannelMessagesResponse struct { state protoimpl.MessageState `protogen:"open.v1"` // Pre-rendered HTML fragment ready to be injected into .main_block. - // Image URLs have been rewritten to call ProxyImage; dates have been - // converted to the Persian calendar. + // Dates have been converted to the Persian calendar. By default + // and background-image:url(...) carry inline + // "data:image/jpeg;base64,…" payloads; with + // disable_inline_images=true they carry "proxy.php?url=" + // placeholders. Html string `protobuf:"bytes,1,opt,name=html,proto3" json:"html,omitempty"` - // Channel avatar URL embedded in the header (only set when before == 0). + // Channel avatar embedded in the header (only set when before == 0). + // "data:image/jpeg;base64,…" by default, "proxy.php?url=" + // placeholder when disable_inline_images=true. ChannelAvatar string `protobuf:"bytes,2,opt,name=channel_avatar,json=channelAvatar,proto3" json:"channel_avatar,omitempty"` // Last post id seen on the page (mirrors the lastread_ cookie). LastPostId int64 `protobuf:"varint,3,opt,name=last_post_id,json=lastPostId,proto3" json:"last_post_id,omitempty"` @@ -511,11 +543,12 @@ var File_v2_ezytel_ezytel_proto protoreflect.FileDescriptor const file_v2_ezytel_ezytel_proto_rawDesc = "" + "\n" + - "\x16v2/ezytel/ezytel.proto\x12\x06ezytel\"P\n" + + "\x16v2/ezytel/ezytel.proto\x12\x06ezytel\"\x84\x01\n" + "\x12ChannelInfoRequest\x12\x1d\n" + "\n" + "channel_id\x18\x01 \x01(\tR\tchannelId\x12\x1b\n" + - "\tlast_read\x18\x02 \x01(\x03R\blastRead\"\xdd\x01\n" + + "\tlast_read\x18\x02 \x01(\x03R\blastRead\x122\n" + + "\x15disable_inline_images\x18\x03 \x01(\bR\x13disableInlineImages\"\xdd\x01\n" + "\vChannelInfo\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12 \n" + "\vdescription\x18\x02 \x01(\tR\vdescription\x12\x1f\n" + @@ -526,11 +559,12 @@ const file_v2_ezytel_ezytel_proto_rawDesc = "" + "\x06newmsg\x18\x06 \x01(\tR\x06newmsg\x12 \n" + "\flast_post_id\x18\a \x01(\x03R\n" + "lastPostId\x12\x0e\n" + - "\x02ok\x18\b \x01(\bR\x02ok\"O\n" + + "\x02ok\x18\b \x01(\bR\x02ok\"\x83\x01\n" + "\x16ChannelMessagesRequest\x12\x1d\n" + "\n" + "channel_id\x18\x01 \x01(\tR\tchannelId\x12\x16\n" + - "\x06before\x18\x02 \x01(\x03R\x06before\"v\n" + + "\x06before\x18\x02 \x01(\x03R\x06before\x122\n" + + "\x15disable_inline_images\x18\x03 \x01(\bR\x13disableInlineImages\"v\n" + "\x17ChannelMessagesResponse\x12\x12\n" + "\x04html\x18\x01 \x01(\tR\x04html\x12%\n" + "\x0echannel_avatar\x18\x02 \x01(\tR\rchannelAvatar\x12 \n" + diff --git a/v2/ezytel/ezytel.proto b/v2/ezytel/ezytel.proto index a0d9e49a0..21a68ac40 100644 --- a/v2/ezytel/ezytel.proto +++ b/v2/ezytel/ezytel.proto @@ -12,14 +12,21 @@ message ChannelInfoRequest { // Last post id the client has already read; used to compute the // unread badge (newmsg). 0 means "everything is unread". int64 last_read = 2; + // By default avatar_path in the response carries a + // "data:image/jpeg;base64,…" URI ready to drop into . + // Set this to true to opt out and receive the legacy + // "cache/.jpg" server-side file path instead. + bool disable_inline_images = 3; } message ChannelInfo { string name = 1; // Plain-text preview of the most recent post, or "فایل" if it is a media post. string description = 2; - // Local cache path of the avatar JPEG, relative to the cache dir - // returned by GetCacheDir(). Empty if the avatar could not be fetched. + // By default this is a "data:image/jpeg;base64,…" URI ready for + // . When the request set disable_inline_images=true it is + // the legacy "cache/.jpg" file path relative to GetCacheDir(). + // Empty if the avatar could not be fetched. string avatar_path = 3; // Unix epoch seconds of the most recent post. int64 date = 4; @@ -40,14 +47,25 @@ message ChannelMessagesRequest { // 0 = latest page; otherwise the "before" cursor returned by Telegram // (the data-before attribute of messages_more_wrap). int64 before = 2; + // By default every image URL in the returned html and the + // channel_avatar field is a "data:image/jpeg;base64,…" URI ready + // to render. Set this to true to opt out: html and channel_avatar + // will both carry the legacy "proxy.php?url=" placeholder + // form, requiring per-image ProxyImage calls. + bool disable_inline_images = 3; } message ChannelMessagesResponse { // Pre-rendered HTML fragment ready to be injected into .main_block. - // Image URLs have been rewritten to call ProxyImage; dates have been - // converted to the Persian calendar. + // Dates have been converted to the Persian calendar. By default + // and background-image:url(...) carry inline + // "data:image/jpeg;base64,…" payloads; with + // disable_inline_images=true they carry "proxy.php?url=" + // placeholders. string html = 1; - // Channel avatar URL embedded in the header (only set when before == 0). + // Channel avatar embedded in the header (only set when before == 0). + // "data:image/jpeg;base64,…" by default, "proxy.php?url=" + // placeholder when disable_inline_images=true. string channel_avatar = 2; // Last post id seen on the page (mirrors the lastread_ cookie). int64 last_post_id = 3; diff --git a/v2/ezytel/ezytel_service_imp.go b/v2/ezytel/ezytel_service_imp.go index 8b67cb76f..4438b5351 100644 --- a/v2/ezytel/ezytel_service_imp.go +++ b/v2/ezytel/ezytel_service_imp.go @@ -4,6 +4,7 @@ import ( "context" "crypto/md5" "crypto/tls" + "encoding/base64" "encoding/hex" "encoding/json" "fmt" @@ -134,10 +135,123 @@ func (s *EzytelService) GetChannelMessages(ctx context.Context, in *ChannelMessa } } + if !in.DisableInlineImages { + // chanPic at this point is "proxy.php?url=" because it + // was captured after rewriteImgSources. Resolve it once so + // resp.ChannelAvatar is also a data: URI; the same placeholder + // inside content gets substituted by inlineProxyPlaceholders. + if chanPicURI := s.inlineProxyPlaceholder(ctx, chanPic); chanPicURI != "" { + resp.ChannelAvatar = chanPicURI + } + content = s.inlineProxyPlaceholders(ctx, content) + } + resp.Html = content return resp, nil } +// inlineProxyPlaceholder resolves a single "proxy.php?url=" +// reference into a data: URI. Returns "" on any failure (caller +// should keep the original value). +func (s *EzytelService) inlineProxyPlaceholder(ctx context.Context, ref string) string { + const prefix = "proxy.php?url=" + idx := strings.Index(ref, prefix) + if idx < 0 { + return "" + } + hexStr := ref[idx+len(prefix):] + raw, err := hex.DecodeString(hexStr) + if err != nil { + return "" + } + data, _, ferr := s.fetchImageBytes(ctx, "https://"+string(raw)) + if ferr != nil { + return "" + } + return dataURL(data, "image/jpeg") +} + +// inlineProxyPlaceholders walks the post-rewrite HTML and replaces every +// "proxy.php?url=" reference with a "data:image/jpeg;base64,…" +// URI. Fetches run in a bounded worker pool so a 20-image page does not +// serialise downloads. On per-image failure the placeholder is left +// alone so the rest of the page still renders. +func (s *EzytelService) inlineProxyPlaceholders(ctx context.Context, content string) string { + hexes := extractProxyHexes(content) + if len(hexes) == 0 { + return content + } + type result struct { + hex string + uri string + } + jobs := make(chan string, len(hexes)) + results := make(chan result, len(hexes)) + const workers = 8 + var wg sync.WaitGroup + for i := 0; i < workers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for h := range jobs { + raw, err := hex.DecodeString(h) + if err != nil { + continue + } + data, _, ferr := s.fetchImageBytes(ctx, "https://"+string(raw)) + if ferr != nil { + continue + } + results <- result{hex: h, uri: dataURL(data, "image/jpeg")} + } + }() + } + for h := range hexes { + jobs <- h + } + close(jobs) + wg.Wait() + close(results) + + pairs := make([]string, 0, 2*len(hexes)) + for r := range results { + pairs = append(pairs, "proxy.php?url="+r.hex, r.uri) + } + if len(pairs) == 0 { + return content + } + return strings.NewReplacer(pairs...).Replace(content) +} + +// extractProxyHexes returns the set of distinct hex bodies referenced +// by "proxy.php?url=" in s. The hex token is the run of [0-9a-f] +// (case-insensitive) immediately after the prefix. +func extractProxyHexes(s string) map[string]struct{} { + out := map[string]struct{}{} + const prefix = "proxy.php?url=" + i := 0 + for { + idx := strings.Index(s[i:], prefix) + if idx < 0 { + return out + } + start := i + idx + len(prefix) + end := start + for end < len(s) { + c := s[end] + if (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F') { + end++ + continue + } + break + } + if end > start { + out[strings.ToLower(s[start:end])] = struct{}{} + } + i = end + } +} + // --------------------------------------------------------------------------- // RPC: GetChannelInfo — port of json_info() in libs.php. // --------------------------------------------------------------------------- @@ -159,6 +273,15 @@ func (s *EzytelService) GetChannelInfo(ctx context.Context, in *ChannelInfoReque if jerr := json.Unmarshal(data, &snap); jerr == nil { snap.Newmsg = "OFF" snap.Ok = false + // Snapshot stores the file-path form. Re-encode to a + // data: URI when the caller wants inline images. + if !in.DisableInlineImages && strings.HasPrefix(snap.AvatarPath, "cache/") { + if b, rerr := os.ReadFile(filepath.Join(s.cacheDir, strings.TrimPrefix(snap.AvatarPath, "cache/"))); rerr == nil { + snap.AvatarPath = dataURL(b, "image/jpeg") + } else { + snap.AvatarPath = "" + } + } return &snap, nil } } @@ -175,9 +298,12 @@ func (s *EzytelService) GetChannelInfo(ctx context.Context, in *ChannelInfoReque pic := strFind(html, []string{`.jpg" cache filename. Shared by ProxyImage, GetChannelInfo +// and GetChannelMessages so all three reuse the same cache. +func (s *EzytelService) fetchImageBytes(ctx context.Context, src string) ([]byte, string, error) { + hash := md5sum(strings.TrimPrefix(strings.TrimPrefix(src, "https://"), "http://")) + cacheName := hash + ".jpg" + if err := s.curlDownload(ctx, src, cacheName); err != nil { + return nil, cacheName, err + } + data, err := os.ReadFile(filepath.Join(s.cacheDir, cacheName)) + if err != nil { + return nil, cacheName, err + } + return data, cacheName, nil +} + +// dataURL builds a "data:;base64," URI ready to drop into +// . Empty contentType defaults to image/jpeg. +func dataURL(data []byte, contentType string) string { + if contentType == "" { + contentType = "image/jpeg" + } + return "data:" + contentType + ";base64," + base64.StdEncoding.EncodeToString(data) +} + // --------------------------------------------------------------------------- // RPC: ParseChannels — port of file_parse() in libs.php. // --------------------------------------------------------------------------- diff --git a/v2/ezytel/ezytel_test.go b/v2/ezytel/ezytel_test.go index fcd200e13..7056a4472 100644 --- a/v2/ezytel/ezytel_test.go +++ b/v2/ezytel/ezytel_test.go @@ -1,8 +1,14 @@ package ezytel import ( + "bytes" "context" + "encoding/base64" "encoding/hex" + "io" + "net/http" + "strings" + "sync/atomic" "testing" "time" ) @@ -86,6 +92,109 @@ func TestStripTags(t *testing.T) { } } +// minimal 1x1 jpeg used by the stub transport +var fakeJPEG = []byte{ + 0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10, 'J', 'F', 'I', 'F', 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, + 0x00, 0x01, 0x00, 0x00, 0xff, 0xd9, +} + +type stubRT struct { + body []byte + count int32 +} + +func (s *stubRT) RoundTrip(_ *http.Request) (*http.Response, error) { + atomic.AddInt32(&s.count, 1) + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(s.body)), + Header: http.Header{}, + }, nil +} + +func TestDataURL(t *testing.T) { + got := dataURL([]byte{0xff, 0xd8, 0xff}, "") + want := "data:image/jpeg;base64," + base64.StdEncoding.EncodeToString([]byte{0xff, 0xd8, 0xff}) + if got != want { + t.Fatalf("dataURL: got %q want %q", got, want) + } +} + +func TestExtractProxyHexes(t *testing.T) { + html := ` and url('proxy.php?url=cafe01') and proxy.php?url=` // trailing empty + hexes := extractProxyHexes(html) + if _, ok := hexes["deadbeef"]; !ok { + t.Fatalf("missing deadbeef in %v", hexes) + } + if _, ok := hexes["cafe01"]; !ok { + t.Fatalf("missing cafe01 in %v", hexes) + } + if len(hexes) != 2 { + t.Fatalf("unexpected entries: %v", hexes) + } +} + +func TestInlineProxyPlaceholders(t *testing.T) { + s := NewEzytelService(t.TempDir()) + rt := &stubRT{body: fakeJPEG} + s.client = &http.Client{Transport: rt} + + hexedA := hex.EncodeToString([]byte("cdn.example.com/a.jpg")) + hexedB := hex.EncodeToString([]byte("cdn.example.com/b.jpg")) + html := `` + + out := s.inlineProxyPlaceholders(context.Background(), html) + if strings.Contains(out, "proxy.php?url=") { + t.Fatalf("placeholders not replaced: %s", out) + } + want := "data:image/jpeg;base64," + base64.StdEncoding.EncodeToString(fakeJPEG) + if !strings.Contains(out, want) { + t.Fatalf("data URI missing in output: %s", out) + } + if got := atomic.LoadInt32(&rt.count); got != 2 { + t.Fatalf("expected 2 fetches, got %d", got) + } +} + +func TestDisableInlineImagesNoFetch(t *testing.T) { + s := NewEzytelService(t.TempDir()) + rt := &stubRT{body: fakeJPEG} + s.client = &http.Client{Transport: rt} + hexed := hex.EncodeToString([]byte("cdn.example.com/a.jpg")) + html := `` + + // Mirror the legacy short-circuit in GetChannelMessages: when the + // flag is true, inlineProxyPlaceholders is not called at all and + // no fetches happen. + in := &ChannelMessagesRequest{ChannelId: "x", DisableInlineImages: true} + if !in.DisableInlineImages { + t.Fatal("flag setup wrong") + } + if got := atomic.LoadInt32(&rt.count); got != 0 { + t.Fatalf("unexpected fetch before any work: %d", got) + } + if !strings.Contains(html, "proxy.php?url="+hexed) { + t.Fatal("placeholder missing in fixture") + } + if strings.Contains(html, "data:image") { + t.Fatalf("legacy html should not contain data: URI: %s", html) + } +} + +func TestInlineProxyPlaceholderSingle(t *testing.T) { + s := NewEzytelService(t.TempDir()) + s.client = &http.Client{Transport: &stubRT{body: fakeJPEG}} + hexed := hex.EncodeToString([]byte("cdn.example.com/avatar.jpg")) + got := s.inlineProxyPlaceholder(context.Background(), "proxy.php?url="+hexed) + want := "data:image/jpeg;base64," + base64.StdEncoding.EncodeToString(fakeJPEG) + if got != want { + t.Fatalf("got %q want %q", got, want) + } + if got := s.inlineProxyPlaceholder(context.Background(), "https://no.placeholder/x"); got != "" { + t.Fatalf("expected empty for non-placeholder, got %q", got) + } +} + func contains(s, sub string) bool { return len(sub) == 0 || (len(s) >= len(sub) && indexOf(s, sub) >= 0) }