diff --git a/docker-compose.selfhost.yml b/docker-compose.selfhost.yml index 0ef30d0034..210b773ee0 100644 --- a/docker-compose.selfhost.yml +++ b/docker-compose.selfhost.yml @@ -67,6 +67,16 @@ services: CLOUDFRONT_DOMAIN: ${CLOUDFRONT_DOMAIN:-} CLOUDFRONT_KEY_PAIR_ID: ${CLOUDFRONT_KEY_PAIR_ID:-} CLOUDFRONT_PRIVATE_KEY: ${CLOUDFRONT_PRIVATE_KEY:-} + # Aliyun OSS native SDK settings (takes precedence over S3-compatible when OSS_BUCKET is set) + OSS_BUCKET: ${OSS_BUCKET:-} + OSS_REGION: ${OSS_REGION:-cn-hangzhou} + ALIBABA_CLOUD_ACCESS_KEY_ID: ${ALIBABA_CLOUD_ACCESS_KEY_ID:-} + ALIBABA_CLOUD_ACCESS_KEY_SECRET: ${ALIBABA_CLOUD_ACCESS_KEY_SECRET:-} + OSS_CDN_DOMAIN: ${OSS_CDN_DOMAIN:-} + OSS_CDN_AUTH_KEY: ${OSS_CDN_AUTH_KEY:-} + OSS_ENDPOINT: ${OSS_ENDPOINT:-} + # Set to "true" to disable OSS presigned URL generation (returns empty URL instead) + OSS_PRESIGN_DISABLED: ${OSS_PRESIGN_DISABLED:-false} COOKIE_DOMAIN: ${COOKIE_DOMAIN:-} APP_ENV: ${APP_ENV:-production} MULTICA_DEV_VERIFICATION_CODE: ${MULTICA_DEV_VERIFICATION_CODE:-} diff --git a/server/cmd/server/router.go b/server/cmd/server/router.go index 9048da2810..4340a4b43c 100644 --- a/server/cmd/server/router.go +++ b/server/cmd/server/router.go @@ -116,11 +116,13 @@ func NewRouterWithOptions(pool *pgxpool.Pool, hub *realtime.Hub, bus *events.Bus daemonHub = daemonws.NewHub() } - // Initialize storage with S3 as primary, fallback to local + // Initialize storage with S3 as primary, OSS as secondary, fallback to local var store storage.Storage s3 := storage.NewS3StorageFromEnv() if s3 != nil { store = s3 + } else if oss := storage.NewOSSStorageFromEnv(); oss != nil { + store = oss } else { local := storage.NewLocalStorageFromEnv() if local != nil { diff --git a/server/go.mod b/server/go.mod index 89108c14e9..e34244483f 100644 --- a/server/go.mod +++ b/server/go.mod @@ -3,6 +3,7 @@ module github.com/multica-ai/multica/server go 1.26.1 require ( + github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.5.1 github.com/aws/aws-sdk-go-v2 v1.41.5 github.com/aws/aws-sdk-go-v2/config v1.32.13 github.com/aws/aws-sdk-go-v2/credentials v1.19.13 @@ -59,5 +60,6 @@ require ( golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.35.0 // indirect golang.org/x/text v0.35.0 // indirect + golang.org/x/time v0.4.0 // indirect google.golang.org/protobuf v1.36.8 // indirect ) diff --git a/server/go.sum b/server/go.sum index 1878640462..75a80f83e8 100644 --- a/server/go.sum +++ b/server/go.sum @@ -1,3 +1,5 @@ +github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.5.1 h1:vtiFd0hhPAbyYJjztl0wYUq/PqEGkIlDmVuTIy6zw8Y= +github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.5.1/go.mod h1:FTzydeQVmR24FI0D6XWUOMKckjXehM/jgMn1xC+DA9M= github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY= github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o= @@ -139,6 +141,8 @@ golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= +golang.org/x/time v0.4.0 h1:Z81tqI5ddIoXDPvVQ7/7CC9TnLM7ubaFG2qXYd5BbYY= +golang.org/x/time v0.4.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/server/internal/storage/oss.go b/server/internal/storage/oss.go new file mode 100644 index 0000000000..5835143c4b --- /dev/null +++ b/server/internal/storage/oss.go @@ -0,0 +1,234 @@ +package storage + +import ( + "bytes" + "context" + "crypto/md5" + "fmt" + "io" + "log/slog" + "net/url" + "os" + "strings" + "time" + + "github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss" + "github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/credentials" +) + +type OSSStorage struct { + client *oss.Client + bucket string + region string + cdnDomain string + cdnAuthKey string + endpointURL string + staticDomain string + presignDisabled bool +} + +// NewOSSStorageFromEnv creates an OSSStorage from environment variables. +// Returns nil if OSS_BUCKET is not set. +// +// Environment variables: +// - OSS_BUCKET (required) +// - OSS_REGION (required, e.g. cn-hangzhou) +// - ALIBABA_CLOUD_ACCESS_KEY_ID / ALIBABA_CLOUD_ACCESS_KEY_SECRET (optional; falls back to ECS RAM role) +// - OSS_CDN_DOMAIN (optional) +// - OSS_CDN_AUTH_KEY (optional; Alibaba Cloud CDN URL Auth Type A private key — enables CDN signed URLs) +// - OSS_ENDPOINT (optional, custom endpoint for internal/VPC access) +// - STATIC_DOMAIN (optional; hostname for the auth-redirect proxy route) +// - OSS_PRESIGN_DISABLED (optional; set to "true" or "1" to disable presigned URL generation) +func NewOSSStorageFromEnv() *OSSStorage { + bucket := os.Getenv("OSS_BUCKET") + if bucket == "" { + slog.Info("OSS_BUCKET not set, OSS upload disabled") + return nil + } + + region := os.Getenv("OSS_REGION") + if region == "" { + slog.Warn("OSS_REGION not set, OSS upload disabled") + return nil + } + + cfg := oss.LoadDefaultConfig().WithRegion(region) + + accessKey := os.Getenv("ALIBABA_CLOUD_ACCESS_KEY_ID") + secretKey := os.Getenv("ALIBABA_CLOUD_ACCESS_KEY_SECRET") + if accessKey != "" && secretKey != "" { + cfg = cfg.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKey, secretKey)) + } else { + cfg = cfg.WithCredentialsProvider(credentials.NewEcsRoleCredentialsProvider()) + } + + endpointURL := os.Getenv("OSS_ENDPOINT") + if endpointURL != "" { + cfg = cfg.WithEndpoint(endpointURL) + } + + cdnDomain := os.Getenv("OSS_CDN_DOMAIN") + cdnAuthKey := os.Getenv("OSS_CDN_AUTH_KEY") + staticDomain := strings.TrimSpace(os.Getenv("STATIC_DOMAIN")) + + presignDisabledVal := strings.TrimSpace(os.Getenv("OSS_PRESIGN_DISABLED")) + presignDisabled := presignDisabledVal == "true" || presignDisabledVal == "1" + + slog.Info("OSS storage initialized", + "bucket", bucket, + "region", region, + "cdn_domain", cdnDomain, + "cdn_auth", cdnAuthKey != "", + "endpoint", endpointURL, + "static_domain", staticDomain, + "presign_disabled", presignDisabled, + ) + return &OSSStorage{ + client: oss.NewClient(cfg), + bucket: bucket, + region: region, + cdnDomain: cdnDomain, + cdnAuthKey: cdnAuthKey, + endpointURL: endpointURL, + staticDomain: staticDomain, + presignDisabled: presignDisabled, + } +} + +func (o *OSSStorage) CdnDomain() string { + return o.cdnDomain +} + +// KeyFromURL extracts the OSS object key from a CDN or bucket URL. +func (o *OSSStorage) KeyFromURL(rawURL string) string { + prefixes := make([]string, 0, 3) + if o.cdnDomain != "" { + prefixes = append(prefixes, "https://"+o.cdnDomain+"/") + } + if o.endpointURL != "" { + prefixes = append(prefixes, strings.TrimRight(o.endpointURL, "/")+"/"+o.bucket+"/") + } + // virtual-hosted-style: https://.oss-.aliyuncs.com/ + prefixes = append(prefixes, "https://"+o.bucket+".oss-"+o.region+".aliyuncs.com/") + + for _, prefix := range prefixes { + if strings.HasPrefix(rawURL, prefix) { + return strings.TrimPrefix(rawURL, prefix) + } + } + // Generic fallback: extract full path from any URL so directory structure is preserved. + if u, err := url.Parse(rawURL); err == nil && u.Path != "" { + return strings.TrimPrefix(u.Path, "/") + } + if i := strings.LastIndex(rawURL, "/"); i >= 0 { + return rawURL[i+1:] + } + return rawURL +} + +// PresignGetURL generates a time-limited signed URL for direct object download. +// Returns an empty string without error when OSS_PRESIGN_DISABLED is set. +// +// When OSS_CDN_DOMAIN and OSS_CDN_AUTH_KEY are both set, it generates an +// Alibaba Cloud CDN URL Authentication (Type A) signed URL. Otherwise it falls +// back to an OSS presigned URL that points directly to the bucket endpoint. +func (o *OSSStorage) PresignGetURL(ctx context.Context, key string, expiry time.Duration) (string, error) { + if o.presignDisabled { + return "", nil + } + if key == "" { + return "", fmt.Errorf("oss PresignGetURL: empty key") + } + if o.cdnDomain != "" && o.cdnAuthKey != "" { + return o.signCDNURL(key, expiry), nil + } + result, err := o.client.Presign(ctx, &oss.GetObjectRequest{ + Bucket: oss.Ptr(o.bucket), + Key: oss.Ptr(key), + }, oss.PresignExpires(expiry)) + if err != nil { + return "", fmt.Errorf("oss presign: %w", err) + } + return result.URL, nil +} + +// signCDNURL generates an Alibaba Cloud CDN URL Authentication Type A URL. +// +// Type A format: https:///?auth_key=--- +// MD5 input: /---- +func (o *OSSStorage) signCDNURL(key string, expiry time.Duration) string { + uri := "/" + key + timestamp := time.Now().Add(expiry).Unix() + rand := "0" + uid := "0" + plain := fmt.Sprintf("%s-%d-%s-%s-%s", uri, timestamp, rand, uid, o.cdnAuthKey) + hash := fmt.Sprintf("%x", md5.Sum([]byte(plain))) + authKey := fmt.Sprintf("%d-%s-%s-%s", timestamp, rand, uid, hash) + return fmt.Sprintf("https://%s%s?auth_key=%s", o.cdnDomain, uri, authKey) +} + +func (o *OSSStorage) GetReader(ctx context.Context, key string) (io.ReadCloser, error) { + if key == "" { + return nil, fmt.Errorf("oss GetReader: empty key") + } + out, err := o.client.GetObject(ctx, &oss.GetObjectRequest{ + Bucket: oss.Ptr(o.bucket), + Key: oss.Ptr(key), + }) + if err != nil { + return nil, fmt.Errorf("oss GetObject: %w", err) + } + return out.Body, nil +} + +func (o *OSSStorage) Delete(ctx context.Context, key string) { + if key == "" { + return + } + _, err := o.client.DeleteObject(ctx, &oss.DeleteObjectRequest{ + Bucket: oss.Ptr(o.bucket), + Key: oss.Ptr(key), + }) + if err != nil { + slog.Error("oss DeleteObject failed", "key", key, "error", err) + } +} + +func (o *OSSStorage) DeleteKeys(ctx context.Context, keys []string) { + for _, key := range keys { + o.Delete(ctx, key) + } +} + +func (o *OSSStorage) Upload(ctx context.Context, key string, data []byte, contentType string, filename string) (string, error) { + safe := sanitizeFilename(filename) + disposition := "attachment" + if isInlineContentType(contentType) { + disposition = "inline" + } + _, err := o.client.PutObject(ctx, &oss.PutObjectRequest{ + Bucket: oss.Ptr(o.bucket), + Key: oss.Ptr(key), + Body: bytes.NewReader(data), + ContentType: oss.Ptr(contentType), + ContentDisposition: oss.Ptr(fmt.Sprintf(`%s; filename="%s"`, disposition, safe)), + CacheControl: oss.Ptr("max-age=432000,public"), + }) + if err != nil { + return "", fmt.Errorf("oss PutObject: %w", err) + } + return o.uploadedURL(key), nil +} + +func (o *OSSStorage) uploadedURL(key string) string { + if o.staticDomain != "" { + return fmt.Sprintf("https://%s/%s", o.staticDomain, key) + } + if o.cdnDomain != "" { + return fmt.Sprintf("https://%s/%s", o.cdnDomain, key) + } + if o.endpointURL != "" { + return fmt.Sprintf("%s/%s/%s", strings.TrimRight(o.endpointURL, "/"), o.bucket, key) + } + return fmt.Sprintf("https://%s.oss-%s.aliyuncs.com/%s", o.bucket, o.region, key) +}