diff --git a/cmd/history2git15/main.go b/cmd/history2git15/main.go index e9c4cdc4..f79f2bda 100644 --- a/cmd/history2git15/main.go +++ b/cmd/history2git15/main.go @@ -11,6 +11,7 @@ import ( "time" "github.com/ddvk/rmfakecloud/internal/config" + "github.com/ddvk/rmfakecloud/internal/storage" "github.com/ddvk/rmfakecloud/internal/storage/fs" "github.com/ddvk/rmfakecloud/internal/storage/models" "github.com/ddvk/rmfakecloud/internal/ui/viewmodel" @@ -86,14 +87,14 @@ func main() { cfg.DataDir = path.Dir(path.Dir(userdirectory)) filesystem := fs.NewStorage(cfg) - lbs := filesystem.BlobStorage(userdirectory) + lbs := storage.NewBlobStorer(filesystem, filesystem) if tail != 0 && len(history) > tail { history = history[len(history)-tail:] } for _, h := range history { - tree, err := h.GetHashTree(lbs) + tree, err := h.GetHashTree(lbs.RemoteStorage(path.Base(userdirectory))) if err != nil { log.Fatalf("%s: %s: %s", arg, h.Hash, err.Error()) } diff --git a/cmd/relinkfile15/main.go b/cmd/relinkfile15/main.go index c423c57e..2153dbfb 100644 --- a/cmd/relinkfile15/main.go +++ b/cmd/relinkfile15/main.go @@ -6,6 +6,7 @@ import ( "log" "github.com/ddvk/rmfakecloud/internal/config" + "github.com/ddvk/rmfakecloud/internal/storage" "github.com/ddvk/rmfakecloud/internal/storage/fs" "github.com/ddvk/rmfakecloud/internal/storage/models" ) @@ -26,17 +27,17 @@ func main() { cfg := config.FromEnv() filesystem := fs.NewStorage(cfg) - lbs := filesystem.BlobStorage(userId) + lbs := storage.NewBlobStorer(filesystem, filesystem) h := models.RootHistory{Hash: rootHash} - oldtree, err := h.GetHashTree(lbs) + oldtree, err := h.GetHashTree(lbs.RemoteStorage(userId)) if err != nil { log.Fatalf("%s: %s", h.Hash, err.Error()) } - hash, gen, _ := lbs.GetRootIndex() + hash, gen, _ := lbs.RemoteStorage(userId).GetRootIndex() h = models.RootHistory{Hash: hash, Generation: gen} - curtree, err := h.GetHashTree(lbs) + curtree, err := h.GetHashTree(lbs.RemoteStorage(userId)) for _, doc := range oldtree.Docs { concerned := false @@ -48,7 +49,7 @@ func main() { } if concerned { - err = fs.UpdateTree(curtree, lbs, func(t *models.HashTree) error { + err = storage.UpdateTree(curtree, lbs, userId, func(t *models.HashTree) error { return t.Add(doc) }) if err != nil { diff --git a/internal/app/app.go b/internal/app/app.go index 3f0e6a03..6ff76b33 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -37,7 +37,8 @@ type App struct { docStorer storage.DocumentStorer userStorer storage.UserStorer metaStorer storage.MetadataStorer - blobStorer storage.BlobStorage + rootStorer storage.RootStorer + blobStorer *storage.BlobStorer hub *hub.Hub codeConnector CodeConnector hwrClient *hwr.HWRClient @@ -110,7 +111,6 @@ func (app *App) Stop() { } } - // NewApp constructs an app func NewApp(cfg *config.Config) App { debugMode := log.GetLevel() >= log.DebugLevel @@ -152,13 +152,16 @@ func NewApp(cfg *config.Config) App { router.Use(requestLoggerMiddleware()) } + blobStorer := storage.NewBlobStorer(fsStorage, fsStorage) + app := App{ router: router, cfg: cfg, docStorer: fsStorage, userStorer: fsStorage, metaStorer: fsStorage, - blobStorer: fsStorage, + rootStorer: fsStorage, + blobStorer: blobStorer, hub: ntfHub, codeConnector: codeConnector, hwrClient: &hwr.HWRClient{ @@ -170,10 +173,10 @@ func NewApp(cfg *config.Config) App { app.registerRoutes(router) - uiApp := ui.New(cfg, fsStorage, codeConnector, ntfHub, fsStorage, fsStorage) + uiApp := ui.New(cfg, fsStorage, codeConnector, ntfHub, fsStorage, blobStorer) uiApp.RegisterRoutes(router) - storageapp := fs.NewApp(cfg, fsStorage) + storageapp := storage.NewApp(cfg, fsStorage, fsStorage, fsStorage, fsStorage) storageapp.RegisterRoutes(router) return app diff --git a/internal/app/handlers.go b/internal/app/handlers.go index eb80a5d1..59f1c309 100644 --- a/internal/app/handlers.go +++ b/internal/app/handlers.go @@ -24,7 +24,7 @@ import ( "github.com/ddvk/rmfakecloud/internal/integrations" "github.com/ddvk/rmfakecloud/internal/messages" "github.com/ddvk/rmfakecloud/internal/storage" - "github.com/ddvk/rmfakecloud/internal/storage/fs" + "github.com/ddvk/rmfakecloud/internal/storage/models" "github.com/gin-gonic/gin" "github.com/golang-jwt/jwt/v4" "github.com/gorilla/websocket" @@ -222,18 +222,16 @@ type metapayload struct { } func userID(c *gin.Context) string { - //TODO: suppress the warning - //codeql[go/path-injection] - return c.GetString(userIDKey) + return common.SanitizeUid(c.GetString(userIDKey)) } func extFromContentType(contentType string) (string, error) { switch contentType { case "application/epub+zip": - return storage.EpubFileExt, nil + return models.EpubFileExt, nil case "application/pdf": - return storage.PdfFileExt, nil + return models.PdfFileExt, nil } return "", fmt.Errorf("unsupported content type %s", contentType) } @@ -748,7 +746,7 @@ func (app *App) syncUpdateRootV3(c *gin.Context) { } uid := userID(c) - newgeneration, err := app.blobStorer.StoreBlob(uid, RootHash, bytes.NewBufferString(rootv3.Hash), rootv3.Generation) + newgeneration, err := app.rootStorer.UpdateRoot(uid, bytes.NewBufferString(rootv3.Hash), rootv3.Generation) if err != nil { log.Error(err) c.AbortWithStatus(http.StatusInternalServerError) @@ -794,8 +792,8 @@ func crcJSON(c *gin.Context, status int, msg any) { func (app *App) syncGetRootV3(c *gin.Context) { uid := userID(c) - reader, generation, _, _, err := app.blobStorer.LoadBlob(uid, RootHash) - if err == fs.ErrorNotFound { + roothash, generation, err := app.rootStorer.GetRootIndex(uid) + if err == storage.ErrorNotFound { log.Warn("No root file found, assuming this is a new account") c.JSON(http.StatusNotFound, gin.H{"message": "root not found"}) return @@ -807,23 +805,16 @@ func (app *App) syncGetRootV3(c *gin.Context) { return } - roothash, err := io.ReadAll(reader) - if err != nil { - log.Error(err) - c.AbortWithStatus(http.StatusInternalServerError) - return - } - c.JSON(http.StatusOK, messages.SyncRootV3Response{ Generation: generation, - Hash: string(roothash), + Hash: roothash, }) } func (app *App) syncGetRootV4(c *gin.Context) { uid := userID(c) - reader, generation, _, _, err := app.blobStorer.LoadBlob(uid, RootHash) - if err == fs.ErrorNotFound { + roothash, generation, err := app.rootStorer.GetRootIndex(uid) + if err == storage.ErrorNotFound { log.Warn("No root file found, assuming this is a new account") crcJSON(c, http.StatusOK, messages.SyncRootV4Response{ SchemaVersion: SchemaVersion, @@ -837,12 +828,6 @@ func (app *App) syncGetRootV4(c *gin.Context) { return } - roothash, err := io.ReadAll(reader) - if err != nil { - log.Error(err) - c.AbortWithStatus(http.StatusInternalServerError) - return - } crcJSON(c, http.StatusOK, messages.SyncRootV4Response{ Generation: generation, Hash: string(roothash), @@ -883,8 +868,8 @@ func (app *App) blobStorageRead(c *gin.Context) { uid := userID(c) blobID := common.ParamS(fileKey, c) - reader, _, size, hash, err := app.blobStorer.LoadBlob(uid, blobID) - if err == fs.ErrorNotFound { + reader, size, hash, err := app.blobStorer.LoadBlob(uid, blobID) + if err == storage.ErrorNotFound { log.Warn(err) c.AbortWithStatus(http.StatusNotFound) return @@ -908,7 +893,7 @@ func (app *App) blobStorageWrite(c *gin.Context) { hash := c.GetHeader(common.GCPHashHeader) log.Debugf("TODO: check/save etc. write file '%s', hash '%s'", fileName, hash) - _, err := app.blobStorer.StoreBlob(uid, blobID, c.Request.Body, 0) + err := app.blobStorer.StoreBlob(uid, blobID, fileName, hash, c.Request.Body) if err != nil { log.Error(err) c.AbortWithStatus(http.StatusInternalServerError) diff --git a/internal/storage/fs/app.go b/internal/storage/app.go similarity index 64% rename from internal/storage/fs/app.go rename to internal/storage/app.go index 166678aa..6aaba1d8 100644 --- a/internal/storage/fs/app.go +++ b/internal/storage/app.go @@ -1,4 +1,4 @@ -package fs +package storage import ( "crypto/hmac" @@ -22,15 +22,17 @@ const ( tokenParam = "token" generationHeader = "x-goog-generation" generationMatchHeader = "x-goog-if-generation-match" - storageUsage = "storage" - - paramUID = "uid" - paramBlobID = "blobid" - paramExp = "exp" - paramSignature = "signature" - paramScope = "scope" - routeBlob = "/blobstorage" - routeStorage = "/storage" + StorageUsage = "storage" + + ParamUID = "uid" + ParamBlobID = "blobid" + ParamExp = "exp" + ParamSignature = "signature" + ParamScope = "scope" + RouteBlob = "/blobstorage" + RouteStorage = "/storage" + + RootBlob = "root" ) // ErrorNotFound not found @@ -41,28 +43,32 @@ var ErrorWrongGeneration = errors.New("wrong generation") // App file system document storage type App struct { - cfg *config.Config - fs *FileSystemStorage + cfg *config.Config + docStorer DocumentStorer + userStorer UserStorer + rootStorer RootStorer + blobStorer BlobStorage } // NewApp StorageApp various storage routes -func NewApp(cfg *config.Config, fs *FileSystemStorage) *App { +func NewApp(cfg *config.Config, docStorer DocumentStorer, userStorer UserStorer, rootStorer RootStorer, blobStorer BlobStorage) *App { staticWrapper := App{ - fs: fs, - cfg: cfg, + cfg: cfg, + docStorer: docStorer, + blobStorer: blobStorer, + rootStorer: rootStorer, } return &staticWrapper } // RegisterRoutes blah func (app *App) RegisterRoutes(router *gin.Engine) { - - router.GET(routeStorage+"/:"+tokenParam, app.downloadDocument) - router.PUT(routeStorage+"/:"+tokenParam, app.uploadDocument) + router.GET(RouteStorage+"/:"+tokenParam, app.downloadDocument) + router.PUT(RouteStorage+"/:"+tokenParam, app.uploadDocument) //sync15 - router.GET(routeBlob, app.downloadBlob) - router.PUT(routeBlob, app.uploadBlob) + router.GET(RouteBlob, app.downloadBlob) + router.PUT(RouteBlob, app.uploadBlob) } func (app *App) parseToken(token string) (*StorageClaim, error) { @@ -71,7 +77,7 @@ func (app *App) parseToken(token string) (*StorageClaim, error) { if err != nil { return nil, err } - if !slices.Contains(claim.Audience, storageUsage) { + if !slices.Contains(claim.Audience, StorageUsage) { return nil, errors.New("not a storage token") } return claim, nil @@ -92,7 +98,7 @@ func (app *App) uploadDocument(c *gin.Context) { body := c.Request.Body defer body.Close() - err = app.fs.StoreDocument(token.UserID, id, body) + err = app.docStorer.StoreDocument(token.UserID, id, body) if err != nil { log.Error(err) c.AbortWithStatus(http.StatusInternalServerError) @@ -115,7 +121,7 @@ func (app *App) downloadDocument(c *gin.Context) { //todo: storage provider log.Info("Requestng Id: ", id) - reader, err := app.fs.GetDocument(token.UserID, id) + reader, err := app.docStorer.GetDocument(token.UserID, id) if err != nil { log.Error(err) @@ -128,12 +134,12 @@ func (app *App) downloadDocument(c *gin.Context) { func (app *App) downloadBlob(c *gin.Context) { //not sanitized, email address etc - uid := c.Query(paramUID) + uid := c.Query(ParamUID) - blobID := common.QueryS(paramBlobID, c) - exp := common.QueryS(paramExp, c) - signature := common.QueryS(paramSignature, c) - scope := common.QueryS(paramScope, c) + blobID := common.QueryS(ParamBlobID, c) + exp := common.QueryS(ParamExp, c) + signature := common.QueryS(ParamSignature, c) + scope := common.QueryS(ParamScope, c) err := VerifyURLParams([]string{uid, blobID, exp, scope}, exp, signature, app.cfg.JWTSecretKey) if err != nil { @@ -154,35 +160,45 @@ func (app *App) downloadBlob(c *gin.Context) { log.Info("Requestng blob: ", blobID) - reader, generation, size, crc32c, err := app.fs.LoadBlob(uid, blobID) - if err != nil { - if err == ErrorNotFound { - c.AbortWithStatus(http.StatusNotFound) + if blobID == RootBlob { + root, generation, err := app.rootStorer.GetRootIndex(uid) + if err != nil && err != ErrorNotFound { + log.Error(err) + c.AbortWithStatus(http.StatusInternalServerError) return } - log.Error(err) - c.AbortWithStatus(http.StatusInternalServerError) - return - } - defer reader.Close() - - common.AddHashHeader(c, crc32c) - if blobID == rootBlob { log.Debug("Sending gen for root: ", generation) c.Header(generationHeader, strconv.FormatInt(generation, 10)) + + c.Data(http.StatusOK, "application/octet-stream", []byte(root)) + } else { + reader, size, crc32c, err := app.blobStorer.LoadBlob(uid, blobID) + if err != nil { + if err == ErrorNotFound { + c.AbortWithStatus(http.StatusNotFound) + return + } + log.Error(err) + c.AbortWithStatus(http.StatusInternalServerError) + return + } + defer reader.Close() + + common.AddHashHeader(c, crc32c) + + c.DataFromReader(http.StatusOK, size, "application/octet-stream", reader, nil) } - c.DataFromReader(http.StatusOK, size, "application/octet-stream", reader, nil) } func (app *App) uploadBlob(c *gin.Context) { //not sanitized, email address etc - uid := c.Query(paramUID) + uid := c.Query(ParamUID) - blobID := common.QueryS(paramBlobID, c) - exp := common.QueryS(paramExp, c) - signature := common.QueryS(paramSignature, c) - scope := common.QueryS(paramScope, c) + blobID := common.QueryS(ParamBlobID, c) + exp := common.QueryS(ParamExp, c) + signature := common.QueryS(ParamSignature, c) + scope := common.QueryS(ParamScope, c) err := VerifyURLParams([]string{uid, blobID, exp, scope}, exp, signature, app.cfg.JWTSecretKey) if err != nil { @@ -205,7 +221,7 @@ func (app *App) uploadBlob(c *gin.Context) { body := c.Request.Body defer body.Close() - generation := int64(0) + var generation int64 gh := c.Request.Header.Get(generationMatchHeader) if gh != "" { log.Info("Client sent generation:", gh) @@ -216,11 +232,17 @@ func (app *App) uploadBlob(c *gin.Context) { } } - newgen, err := app.fs.StoreBlob(uid, blobID, body, generation) + if blobID == RootBlob { + var newgen int64 + newgen, err = app.rootStorer.UpdateRoot(uid, body, generation) + if err == ErrorWrongGeneration { + c.AbortWithStatus(http.StatusPreconditionFailed) + return + } - if err == ErrorWrongGeneration { - c.AbortWithStatus(http.StatusPreconditionFailed) - return + c.Header(generationHeader, strconv.FormatInt(newgen, 10)) + } else { + err = app.blobStorer.StoreBlob(uid, blobID, "", "", body) } if err != nil { @@ -229,7 +251,6 @@ func (app *App) uploadBlob(c *gin.Context) { return } - c.Header(generationHeader, strconv.FormatInt(newgen, 10)) c.JSON(http.StatusOK, gin.H{}) } diff --git a/internal/storage/blobstore.go b/internal/storage/blobstore.go new file mode 100644 index 00000000..f3d5aef9 --- /dev/null +++ b/internal/storage/blobstore.go @@ -0,0 +1,663 @@ +package storage + +import ( + "archive/zip" + "bytes" + "encoding/json" + "errors" + "io" + "os" + "path" + "strings" + "time" + + "github.com/ddvk/rmfakecloud/internal/common" + "github.com/ddvk/rmfakecloud/internal/storage/exporter" + "github.com/ddvk/rmfakecloud/internal/storage/models" + "github.com/google/uuid" + log "github.com/sirupsen/logrus" +) + +type BlobStorer struct { + impl BlobStorage + root RootStorer +} + +func NewBlobStorer(impl BlobStorage, rootStorer RootStorer) *BlobStorer { + return &BlobStorer{impl, rootStorer} +} + +func (bs *BlobStorer) RemoteStorage(uid string) models.RemoteStorage { + return &BlobRemoteStorage{ + bs: bs, + uid: uid, + } +} + +// GetBlobURL return a url for a file to store +func (bs *BlobStorer) GetBlobURL(uid, blobid string, write bool) (string, time.Time, error) { + return bs.impl.GetBlobURL(uid, blobid, write) +} + +// LoadBlob Opens a blob by id +func (bs *BlobStorer) LoadBlob(uid, blobid string) (io.ReadCloser, int64, string, error) { + return bs.impl.LoadBlob(uid, blobid) +} + +// StoreBlob stores a document +func (bs *BlobStorer) StoreBlob(uid, id string, fileName string, hash string, stream io.Reader) error { + return bs.impl.StoreBlob(uid, id, fileName, hash, stream) +} + +func (bs *BlobStorer) GetCachedTree(uid string) (tree *models.HashTree, err error) { + return bs.root.GetCachedTree(uid, bs.RemoteStorage(uid)) +} + +// ExportRmDoc exports a document as a zip of all blobs +func (bs *BlobStorer) ExportRmDoc(uid, docid string) (io.ReadCloser, error) { + tree, err := bs.root.GetCachedTree(uid, bs.RemoteStorage(uid)) + if err != nil { + return nil, err + } + doc, err := tree.FindDoc(docid) + if err != nil { + return nil, err + } + ls := bs.RemoteStorage(uid) + + reader, writer := io.Pipe() + go func() { + zw := zip.NewWriter(writer) + var writeErr error + for _, entry := range doc.Files { + blob, err := ls.GetReader(entry.Hash) + if err != nil { + writeErr = err + break + } + fw, err := zw.Create(entry.EntryName) + if err != nil { + blob.Close() + writeErr = err + break + } + _, err = io.Copy(fw, blob) + blob.Close() + if err != nil { + writeErr = err + break + } + } + if writeErr != nil { + log.Error(writeErr) + zw.Close() + writer.CloseWithError(writeErr) + return + } + zw.Close() + writer.Close() + }() + return reader, nil +} + +// Export exports a document +func (bs *BlobStorer) Export(uid, docid string) (r io.ReadCloser, err error) { + tree, err := bs.root.GetCachedTree(uid, bs.RemoteStorage(uid)) + if err != nil { + return nil, err + } + doc, err := tree.FindDoc(docid) + if err != nil { + return nil, err + } + ls := bs.RemoteStorage(uid) + + archive, err := models.ArchiveFromHashDoc(doc, ls) + if err != nil { + return nil, err + } + reader, writer := io.Pipe() + go func() { + err = exporter.RenderRmapi(archive, writer) + if err != nil { + log.Error(err) + writer.Close() + return + } + writer.Close() + }() + return reader, err +} + +// UpdateBlobDocument updates metadata +func (bs *BlobStorer) UpdateBlobDocument(uid, docID, name, parent string) (err error) { + tree, err := bs.root.GetCachedTree(uid, bs.RemoteStorage(uid)) + if err != nil { + return err + } + + log.Info("updateBlobDocument: ", docID, "new name:", name) + + err = UpdateTree(tree, bs, uid, func(t *models.HashTree) error { + hashDoc, err := tree.FindDoc(docID) + if err != nil { + return err + } + log.Info("updateBlobDocument: ", hashDoc.DocumentName) + + hashDoc.DocumentName = name + hashDoc.Parent = parent + hashDoc.Version++ + + metadataHash, metadataCRC32C, metadataReader, err := hashDoc.MetadataReader() + if err != nil { + return err + } + + err = bs.impl.StoreBlob(uid, metadataHash, hashDoc.DocumentName+models.MetadataFileExt, "crc32c="+metadataCRC32C, metadataReader) + if err != nil { + return err + } + + //update the metadata hash + for _, hashEntry := range hashDoc.Files { + if hashEntry.IsMetadata() { + hashEntry.Hash = metadataHash + break + } + } + hashDoc.Rehash() + hashDocReader, err := hashDoc.IndexReader() + if err != nil { + return err + } + hashDocContent, err := io.ReadAll(hashDocReader) + if err != nil { + return err + } + + crc32cIndex, err := common.CRC32CFromReader(bytes.NewReader(hashDocContent)) + if err != nil { + return err + } + + err = bs.impl.StoreBlob(uid, hashDoc.Hash, hashDoc.DocumentName, "crc32c="+crc32cIndex, bytes.NewReader(hashDocContent)) + if err != nil { + return err + } + + t.Rehash() + return nil + }) + + return err +} + +// DeleteBlobDocument deletes blob document +func (bs *BlobStorer) DeleteBlobDocument(uid, docID string) (err error) { + tree, err := bs.root.GetCachedTree(uid, bs.RemoteStorage(uid)) + if err != nil { + return err + } + + return UpdateTree(tree, bs, uid, func(t *models.HashTree) error { + return tree.Remove(docID) + }) +} + +// CreateBlobFolder creates a new folder +func (bs *BlobStorer) CreateBlobFolder(uid, foldername, parent string) (doc *Document, err error) { + docID := uuid.New().String() + tree, err := bs.root.GetCachedTree(uid, bs.RemoteStorage(uid)) + if err != nil { + return nil, err + } + + log.Info("Creating blob folder ", foldername, " parent: ", parent) + + metadata := models.MetadataFile{ + DocumentName: foldername, + CollectionType: common.CollectionType, + Parent: parent, + Version: 1, + CreatedTime: models.FromTime(time.Now()), + LastModified: models.FromTime(time.Now()), + Synced: true, + MetadataModified: true, + } + + metadataReader, metahash, crc32c, size, err := createMetadataFile(metadata) + if err != nil { + return + } + + log.Info("meta hash: ", metahash) + err = bs.impl.StoreBlob(uid, metahash, "", "crc32c="+crc32c, metadataReader) + if err != nil { + return nil, err + } + + metadataEntry := models.NewHashEntry(metahash, docID+models.MetadataFileExt, size) + hashDoc := models.NewHashDocWithMeta(docID, metadata) + err = hashDoc.AddFile(metadataEntry) + + if err != nil { + return nil, err + } + hashDocReader, err := hashDoc.IndexReader() + if err != nil { + return nil, err + } + + hashDocContent, err := io.ReadAll(hashDocReader) + if err != nil { + return nil, err + } + + crc32c, err = common.CRC32CFromReader(bytes.NewReader(hashDocContent)) + if err != nil { + return nil, err + } + + err = bs.impl.StoreBlob(uid, hashDoc.Hash, "", "crc32c="+crc32c, bytes.NewReader(hashDocContent)) + if err != nil { + return nil, err + } + + err = UpdateTree(tree, bs, uid, func(t *models.HashTree) error { + return t.Add(hashDoc) + }) + + if err != nil { + return nil, err + } + + doc = &Document{ + ID: docID, + Type: common.CollectionType, + Parent: parent, + Name: foldername, + } + return doc, nil +} + +// updates the tree and saves the new root +func UpdateTree(tree *models.HashTree, bs *BlobStorer, uid string, treeMutation func(t *models.HashTree) error) error { + for i := 0; i < 3; i++ { + err := treeMutation(tree) + if err != nil { + return err + } + + rootIndexReader, err := tree.RootIndex() + if err != nil { + return err + } + + rootIndexContent, err := io.ReadAll(rootIndexReader) + if err != nil { + return err + } + + crc32cIndex, err := common.CRC32CFromReader(bytes.NewReader(rootIndexContent)) + if err != nil { + return err + } + + err = bs.impl.StoreBlob(uid, tree.Hash, "roothash", "crc32c="+crc32cIndex, bytes.NewReader(rootIndexContent)) + if err != nil { + return err + } + + gen, err := bs.root.UpdateRoot(uid, bytes.NewBufferString(tree.Hash), tree.Generation) + //the tree has been updated + if err == ErrorWrongGeneration { + tree.Mirror(bs.RemoteStorage(uid)) + continue + } + if err != nil { + return err + } + log.Info("got new root gen ", gen) + tree.Generation = gen + //TODO: concurrency + err = bs.root.SaveCachedTree(uid, tree) + + if err != nil { + return err + } + + return nil + } + return errors.New("could not update") +} + +// CreateBlobDocument creates a new document +func (bs *BlobStorer) CreateBlobDocument(uid, filename, parent string, stream io.Reader) (doc *Document, err error) { + ext := path.Ext(filename) + switch ext { + case models.EpubFileExt, models.PdfFileExt, models.RmDocFileExt: + default: + return nil, errors.New("unsupported extension: " + ext) + } + + if ext == models.RmDocFileExt { + return bs.createFromRmDoc(uid, parent, stream) + } + + docid := uuid.New().String() + docName := strings.TrimSuffix(filename, ext) + + tree, err := bs.root.GetCachedTree(uid, bs.RemoteStorage(uid)) + if err != nil { + return nil, err + } + + log.Info("Creating metadata... parent: ", parent) + + metadata := models.MetadataFile{ + DocumentName: docName, + CollectionType: common.DocumentType, + Parent: parent, + Version: 1, + CreatedTime: models.FromTime(time.Now()), + LastModified: models.FromTime(time.Now()), + Synced: true, + MetadataModified: true, + } + + r, metahash, crc32c, size, err := createMetadataFile(metadata) + if err != nil { + return nil, err + } + + err = bs.impl.StoreBlob(uid, metahash, "", "crc32c="+crc32c, r) + if err != nil { + return nil, err + } + + payloadEntry := models.NewHashEntry(metahash, docid+models.MetadataFileExt, size) + + hashDoc := models.NewHashDocWithMeta(docid, metadata) + hashDoc.PayloadType = docName + + err = hashDoc.AddFile(payloadEntry) + if err != nil { + return + } + + content := models.CreateContent(ext) + + contentReader := strings.NewReader(content) + contentHash, crc32c, size, err := models.Hash(contentReader) + if err != nil { + return + } + _, err = contentReader.Seek(0, io.SeekStart) + if err != nil { + return + } + err = bs.impl.StoreBlob(uid, contentHash, "", "crc32c="+crc32c, contentReader) + if err != nil { + return + } + payloadEntry = models.NewHashEntry(contentHash, docid+models.ContentFileExt, size) + + err = hashDoc.AddFile(payloadEntry) + if err != nil { + return + } + + // given that the payload can be huge + // calculate the hash while streaming the payload to the storage + // then rename it + tmpdoc, err := os.CreateTemp("", "rmfakecloud-upload") + if err != nil { + return + } + defer tmpdoc.Close() + defer os.Remove(tmpdoc.Name()) + + tee := io.TeeReader(stream, tmpdoc) + payloadHash, crc32c, size, err := models.Hash(tee) + if err != nil { + return nil, err + } + tmpdoc.Close() + + // Save payload + fd, err := os.Open(tmpdoc.Name()) + if err != nil { + return nil, err + } + defer fd.Close() + + err = bs.impl.StoreBlob(uid, payloadHash, "", "crc32c="+crc32c, fd) + if err != nil { + return nil, err + } + + payloadEntry = models.NewHashEntry(payloadHash, docid+ext, size) + err = hashDoc.AddFile(payloadEntry) + + if err != nil { + return nil, err + } + + indexReader, err := hashDoc.IndexReader() + if err != nil { + return nil, err + } + + indexContent, err := io.ReadAll(indexReader) + if err != nil { + return nil, err + } + + crc32cIndex, err := common.CRC32CFromReader(bytes.NewReader(indexContent)) + if err != nil { + return nil, err + } + + err = bs.impl.StoreBlob(uid, hashDoc.Hash, "", "crc32c="+crc32cIndex, bytes.NewReader(indexContent)) + if err != nil { + return nil, err + } + + err = UpdateTree(tree, bs, uid, func(t *models.HashTree) error { + return tree.Add(hashDoc) + }) + + if err != nil { + return + } + + doc = &Document{ + ID: docid, + Type: common.DocumentType, + Parent: "", + Name: docName, + } + return +} + +func (bs *BlobStorer) createFromRmDoc(uid, parent string, stream io.Reader) (*Document, error) { + data, err := io.ReadAll(stream) + if err != nil { + return nil, err + } + + zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + if err != nil { + return nil, err + } + + var metadataEntry *zip.File + for _, f := range zr.File { + if strings.HasSuffix(f.Name, models.MetadataFileExt) { + metadataEntry = f + break + } + } + if metadataEntry == nil { + return nil, errors.New("rmdoc: no .metadata file found in archive") + } + + docid := strings.TrimSuffix(metadataEntry.Name, models.MetadataFileExt) + + mr, err := metadataEntry.Open() + if err != nil { + return nil, err + } + metaBytes, err := io.ReadAll(mr) + mr.Close() + if err != nil { + return nil, err + } + + var metadata models.MetadataFile + if err := json.Unmarshal(metaBytes, &metadata); err != nil { + return nil, err + } + + if parent != "" { + metadata.Parent = parent + } + metadata.Synced = true + metadata.MetadataModified = true + + metaReader := bytes.NewReader(metaBytes) + metaHash, metaCRC32C, metaSize, err := models.Hash(metaReader) + if err != nil { + return nil, err + } + metaReader.Seek(0, io.SeekStart) + err = bs.impl.StoreBlob(uid, metaHash, "", "crc32c="+metaCRC32C, metaReader) + if err != nil { + return nil, err + } + + hashDoc := models.NewHashDocWithMeta(docid, metadata) + hashDoc.PayloadType = metadata.DocumentName + + for _, f := range zr.File { + if strings.HasSuffix(f.Name, models.ContentFileExt) { + cr, err := f.Open() + if err == nil { + var contentFile models.ContentFile + contentBytes, err := io.ReadAll(cr) + cr.Close() + if err == nil { + if json.Unmarshal(contentBytes, &contentFile) == nil && contentFile.FileType != "" { + hashDoc.PayloadType = contentFile.FileType + } + } + } + break + } + } + + entry := models.NewHashEntry(metaHash, metadataEntry.Name, metaSize) + if err := hashDoc.AddFile(entry); err != nil { + return nil, err + } + + for _, f := range zr.File { + if f.Name == metadataEntry.Name { + continue + } + + rc, err := f.Open() + if err != nil { + return nil, err + } + fileData, err := io.ReadAll(rc) + rc.Close() + if err != nil { + return nil, err + } + + reader := bytes.NewReader(fileData) + fileHash, fileCRC32C, fileSize, err := models.Hash(reader) + if err != nil { + return nil, err + } + reader.Seek(0, io.SeekStart) + err = bs.impl.StoreBlob(uid, fileHash, "", "crc32c="+fileCRC32C, reader) + if err != nil { + return nil, err + } + + entry := models.NewHashEntry(fileHash, f.Name, fileSize) + if err := hashDoc.AddFile(entry); err != nil { + return nil, err + } + } + + indexReader, err := hashDoc.IndexReader() + if err != nil { + return nil, err + } + + indexContent, err := io.ReadAll(indexReader) + if err != nil { + return nil, err + } + + crc32cIndex, err := common.CRC32CFromReader(bytes.NewReader(indexContent)) + if err != nil { + return nil, err + } + + err = bs.impl.StoreBlob(uid, hashDoc.Hash, "", "crc32c="+crc32cIndex, bytes.NewReader(indexContent)) + if err != nil { + return nil, err + } + + tree, err := bs.root.GetCachedTree(uid, bs.RemoteStorage(uid)) + if err != nil { + return nil, err + } + err = UpdateTree(tree, bs, uid, func(t *models.HashTree) error { + return tree.Add(hashDoc) + }) + if err != nil { + return nil, err + } + + return &Document{ + ID: docid, + Type: metadata.CollectionType, + Parent: metadata.Parent, + Name: metadata.DocumentName, + }, nil +} + +func createMetadataFile(metadata models.MetadataFile) (r io.Reader, filehash string, crc32c string, size int64, err error) { + jsn, err := json.Marshal(metadata) + if err != nil { + return + } + reader := bytes.NewReader(jsn) + filehash, crc32c, size, err = models.Hash(reader) + if err != nil { + return + } + reader.Seek(0, io.SeekStart) + r = reader + return +} + +type BlobRemoteStorage struct { + bs *BlobStorer + uid string +} + +func (brs *BlobRemoteStorage) GetRootIndex() (hash string, generation int64, err error) { + return brs.bs.root.GetRootIndex(brs.uid) +} + +func (brs *BlobRemoteStorage) GetReader(hash string) (io.ReadCloser, error) { + rc, _, _, err := brs.bs.LoadBlob(brs.uid, hash) + return rc, err +} diff --git a/internal/storage/fs/claims.go b/internal/storage/claims.go similarity index 94% rename from internal/storage/fs/claims.go rename to internal/storage/claims.go index 41d6cf6f..daf581c7 100644 --- a/internal/storage/fs/claims.go +++ b/internal/storage/claims.go @@ -1,4 +1,4 @@ -package fs +package storage import "github.com/golang-jwt/jwt/v4" diff --git a/internal/storage/fs/blobstore.go b/internal/storage/fs/blobstore.go index cffa85f0..14dcd960 100644 --- a/internal/storage/fs/blobstore.go +++ b/internal/storage/fs/blobstore.go @@ -1,667 +1,58 @@ package fs import ( - "archive/zip" - "bytes" - "encoding/json" - "errors" "io" "net/url" "os" "path" "strconv" - "strings" "time" - "github.com/danjacques/gofslock/fslock" "github.com/ddvk/rmfakecloud/internal/common" "github.com/ddvk/rmfakecloud/internal/config" "github.com/ddvk/rmfakecloud/internal/storage" - "github.com/ddvk/rmfakecloud/internal/storage/exporter" - "github.com/ddvk/rmfakecloud/internal/storage/models" - "github.com/google/uuid" log "github.com/sirupsen/logrus" ) -const cachedTreeName = ".tree" - -// serves as root modification log and generation number source -const historyFile = ".root.history" -const rootBlob = "root" - -// GetCachedTree returns the cached blob tree for the user -func (fs *FileSystemStorage) GetCachedTree(uid string) (t *models.HashTree, err error) { - blobStorage := &LocalBlobStorage{ - uid: uid, - fs: fs, - } - - cachePath := path.Join(fs.getUserPath(uid), cachedTreeName) - - tree, err := models.LoadTree(cachePath) - if err != nil { - return nil, err - } - tree.SchemaVersion = fs.Cfg.HashSchemaVersion - changed, err := tree.Mirror(blobStorage) - if err != nil { - return nil, err - } - if changed { - err = tree.Save(cachePath) - if err != nil { - return nil, err - } - } - return tree, nil -} - -// SaveCachedTree saves the cached tree -func (fs *FileSystemStorage) SaveCachedTree(uid string, t *models.HashTree) error { - cachePath := path.Join(fs.getUserPath(uid), cachedTreeName) - return t.Save(cachePath) -} - -func (fs *FileSystemStorage) BlobStorage(uid string) *LocalBlobStorage { - return &LocalBlobStorage{ - fs: fs, - uid: uid, - } -} - -// ExportRmDoc exports a document as a zip of all blobs -func (fs *FileSystemStorage) ExportRmDoc(uid, docid string) (io.ReadCloser, error) { - tree, err := fs.GetCachedTree(uid) - if err != nil { - return nil, err - } - doc, err := tree.FindDoc(docid) - if err != nil { - return nil, err - } - ls := fs.BlobStorage(uid) - - reader, writer := io.Pipe() - go func() { - zw := zip.NewWriter(writer) - var writeErr error - for _, entry := range doc.Files { - blob, err := ls.GetReader(entry.Hash) - if err != nil { - writeErr = err - break - } - fw, err := zw.Create(entry.EntryName) - if err != nil { - blob.Close() - writeErr = err - break - } - _, err = io.Copy(fw, blob) - blob.Close() - if err != nil { - writeErr = err - break - } - } - if writeErr != nil { - log.Error(writeErr) - zw.Close() - writer.CloseWithError(writeErr) - return - } - zw.Close() - writer.Close() - }() - return reader, nil -} - -// Export exports a document -func (fs *FileSystemStorage) Export(uid, docid string) (r io.ReadCloser, err error) { - tree, err := fs.GetCachedTree(uid) - if err != nil { - return nil, err - } - doc, err := tree.FindDoc(docid) - if err != nil { - return nil, err - } - ls := fs.BlobStorage(uid) - - archive, err := models.ArchiveFromHashDoc(doc, ls) - if err != nil { - return nil, err - } - reader, writer := io.Pipe() - go func() { - err = exporter.RenderRmapi(archive, writer) - if err != nil { - log.Error(err) - writer.Close() - return - } - writer.Close() - }() - return reader, err -} - -// UpdateBlobDocument updates metadata -func (fs *FileSystemStorage) UpdateBlobDocument(uid, docID, name, parent string) (err error) { - tree, err := fs.GetCachedTree(uid) - if err != nil { - return nil - } - - log.Info("updateBlobDocument: ", docID, "new name:", name) - blobStorage := fs.BlobStorage(uid) - - err = updateTree(tree, blobStorage, func(t *models.HashTree) error { - hashDoc, err := tree.FindDoc(docID) - if err != nil { - return err - } - log.Info("updateBlobDocument: ", hashDoc.DocumentName) - - hashDoc.DocumentName = name - hashDoc.Parent = parent - hashDoc.Version++ - - metadataHash, metadataReader, err := hashDoc.MetadataReader() - if err != nil { - return err - } - - err = blobStorage.Write(metadataHash, metadataReader) - if err != nil { - return err - } - - //update the metadata hash - for _, hashEntry := range hashDoc.Files { - if hashEntry.IsMetadata() { - hashEntry.Hash = metadataHash - break - } - } - hashDoc.Rehash() - hashDocReader, err := hashDoc.IndexReader() - if err != nil { - return err - } - - err = blobStorage.Write(hashDoc.Hash, hashDocReader) - if err != nil { - return err - } - - t.Rehash() - return nil - }) - - return err -} - -// DeleteBlobDocument deletes blob document -func (fs *FileSystemStorage) DeleteBlobDocument(uid, docID string) (err error) { - tree, err := fs.GetCachedTree(uid) - if err != nil { - return nil - } - - blobStorage := fs.BlobStorage(uid) - - return updateTree(tree, blobStorage, func(t *models.HashTree) error { - return tree.Remove(docID) - }) -} - -// CreateBlobFolder creates a new folder -func (fs *FileSystemStorage) CreateBlobFolder(uid, foldername, parent string) (doc *storage.Document, err error) { - docID := uuid.New().String() - tree, err := fs.GetCachedTree(uid) - if err != nil { - return nil, err - } - - log.Info("Creating blob folder ", foldername, " parent: ", parent) - - blobStorage := fs.BlobStorage(uid) - - metadata := models.MetadataFile{ - DocumentName: foldername, - CollectionType: common.CollectionType, - Parent: parent, - Version: 1, - CreatedTime: models.FromTime(time.Now()), - LastModified: models.FromTime(time.Now()), - Synced: true, - MetadataModified: true, - } - - metadataReader, metahash, size, err := createMetadataFile(metadata) - log.Info("meta hash: ", metahash) - err = blobStorage.Write(metahash, metadataReader) - if err != nil { - return nil, err - } - - metadataEntry := models.NewHashEntry(metahash, docID+storage.MetadataFileExt, size) - hashDoc := models.NewHashDocWithMeta(docID, metadata) - err = hashDoc.AddFile(metadataEntry) - - if err != nil { - return nil, err - } - hashDocReader, err := hashDoc.IndexReader() - if err != nil { - return nil, err - } - - err = blobStorage.Write(hashDoc.Hash, hashDocReader) - if err != nil { - return nil, err - } - - err = updateTree(tree, blobStorage, func(t *models.HashTree) error { - return t.Add(hashDoc) - }) - - if err != nil { - return nil, err - } - - doc = &storage.Document{ - ID: docID, - Type: common.CollectionType, - Parent: parent, - Name: foldername, - } - return doc, nil -} - -func UpdateTree(tree *models.HashTree, storage *LocalBlobStorage, treeMutation func(t *models.HashTree) error) error { - return updateTree(tree, storage, treeMutation) -} - -// updates the tree and saves the new root -func updateTree(tree *models.HashTree, storage *LocalBlobStorage, treeMutation func(t *models.HashTree) error) error { - for i := 0; i < 3; i++ { - err := treeMutation(tree) - if err != nil { - return err - } - - rootIndexReader, err := tree.RootIndex() - if err != nil { - return err - } - err = storage.Write(tree.Hash, rootIndexReader) - if err != nil { - return err - } - - gen, err := storage.WriteRootIndex(tree.Generation, tree.Hash) - //the tree has been updated - if err == ErrorWrongGeneration { - tree.Mirror(storage) - continue - } - if err != nil { - return err - } - log.Info("got new root gen ", gen) - tree.Generation = gen - //TODO: concurrency - err = storage.fs.SaveCachedTree(storage.uid, tree) - - if err != nil { - return err - } - - return nil - } - return errors.New("could not update") -} - -// CreateBlobDocument creates a new document -func (fs *FileSystemStorage) CreateBlobDocument(uid, filename, parent string, stream io.Reader) (doc *storage.Document, err error) { - ext := path.Ext(filename) - switch ext { - case storage.EpubFileExt, storage.PdfFileExt, storage.RmDocFileExt: - default: - return nil, errors.New("unsupported extension: " + ext) - } - - if ext == storage.RmDocFileExt { - return fs.createFromRmDoc(uid, parent, stream) - } - - blobPath := fs.getUserBlobPath(uid) - docid := uuid.New().String() - docName := strings.TrimSuffix(filename, ext) - - tree, err := fs.GetCachedTree(uid) - if err != nil { - return nil, err - } - - log.Info("Creating metadata... parent: ", parent) - - metadata := models.MetadataFile{ - DocumentName: docName, - CollectionType: common.DocumentType, - Parent: parent, - Version: 1, - CreatedTime: models.FromTime(time.Now()), - LastModified: models.FromTime(time.Now()), - Synced: true, - MetadataModified: true, - } - - blobStorage := fs.BlobStorage(uid) - r, metahash, size, err := createMetadataFile(metadata) - blobStorage.Write(metahash, r) - if err != nil { - return nil, err - } - - payloadEntry := models.NewHashEntry(metahash, docid+storage.MetadataFileExt, size) - if err != nil { - return - } - - hashDoc := models.NewHashDocWithMeta(docid, metadata) - hashDoc.PayloadType = docName - - err = hashDoc.AddFile(payloadEntry) - if err != nil { - return - } - - content := createContent(ext) - - contentReader := strings.NewReader(content) - contentHash, size, err := models.Hash(contentReader) - if err != nil { - return - } - _, err = contentReader.Seek(0, io.SeekStart) - if err != nil { - return - } - err = blobStorage.Write(contentHash, contentReader) - if err != nil { - return - } - payloadEntry = models.NewHashEntry(contentHash, docid+storage.ContentFileExt, size) - - err = hashDoc.AddFile(payloadEntry) - if err != nil { - return - } - - // given that the payload can be huge - // calculate the hash while streaming the payload to the storage - // then rename it - tmpdoc, err := os.CreateTemp(blobPath, "blob-upload") - if err != nil { - return - } - defer tmpdoc.Close() - defer os.Remove(tmpdoc.Name()) - - tee := io.TeeReader(stream, tmpdoc) - payloadHash, size, err := models.Hash(tee) - if err != nil { - return nil, err - } - tmpdoc.Close() - payloadFilename := path.Join(blobPath, payloadHash) - log.Debug("new payload name: ", payloadFilename) - err = os.Rename(tmpdoc.Name(), payloadFilename) - if err != nil { - return nil, err - } - payloadEntry = models.NewHashEntry(payloadHash, docid+ext, size) - err = hashDoc.AddFile(payloadEntry) - - if err != nil { - return nil, err - } - - indexReader, err := hashDoc.IndexReader() - if err != nil { - return nil, err - } - err = blobStorage.Write(hashDoc.Hash, indexReader) - if err != nil { - return nil, err - } - - err = updateTree(tree, blobStorage, func(t *models.HashTree) error { - return tree.Add(hashDoc) - }) - - if err != nil { - return - } - - doc = &storage.Document{ - ID: docid, - Type: common.DocumentType, - Parent: "", - Name: docName, - } - return -} - -func (fs *FileSystemStorage) createFromRmDoc(uid, parent string, stream io.Reader) (*storage.Document, error) { - data, err := io.ReadAll(stream) - if err != nil { - return nil, err - } - - zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) - if err != nil { - return nil, err - } - - var metadataEntry *zip.File - for _, f := range zr.File { - if strings.HasSuffix(f.Name, storage.MetadataFileExt) { - metadataEntry = f - break - } - } - if metadataEntry == nil { - return nil, errors.New("rmdoc: no .metadata file found in archive") - } - - docid := strings.TrimSuffix(metadataEntry.Name, storage.MetadataFileExt) - - mr, err := metadataEntry.Open() - if err != nil { - return nil, err - } - metaBytes, err := io.ReadAll(mr) - mr.Close() - if err != nil { - return nil, err - } - - var metadata models.MetadataFile - if err := json.Unmarshal(metaBytes, &metadata); err != nil { - return nil, err - } - - if parent != "" { - metadata.Parent = parent - } - metadata.Synced = true - metadata.MetadataModified = true - - blobStorage := fs.BlobStorage(uid) - - metaReader := bytes.NewReader(metaBytes) - metaHash, metaSize, err := models.Hash(metaReader) - if err != nil { - return nil, err - } - metaReader.Seek(0, io.SeekStart) - if err := blobStorage.Write(metaHash, metaReader); err != nil { - return nil, err - } - - hashDoc := models.NewHashDocWithMeta(docid, metadata) - hashDoc.PayloadType = metadata.DocumentName - - for _, f := range zr.File { - if strings.HasSuffix(f.Name, storage.ContentFileExt) { - cr, err := f.Open() - if err == nil { - var contentFile models.ContentFile - contentBytes, err := io.ReadAll(cr) - cr.Close() - if err == nil { - if json.Unmarshal(contentBytes, &contentFile) == nil && contentFile.FileType != "" { - hashDoc.PayloadType = contentFile.FileType - } - } - } - break - } - } - - entry := models.NewHashEntry(metaHash, metadataEntry.Name, metaSize) - if err := hashDoc.AddFile(entry); err != nil { - return nil, err - } - - for _, f := range zr.File { - if f.Name == metadataEntry.Name { - continue - } - - rc, err := f.Open() - if err != nil { - return nil, err - } - fileData, err := io.ReadAll(rc) - rc.Close() - if err != nil { - return nil, err - } - - reader := bytes.NewReader(fileData) - fileHash, fileSize, err := models.Hash(reader) - if err != nil { - return nil, err - } - reader.Seek(0, io.SeekStart) - if err := blobStorage.Write(fileHash, reader); err != nil { - return nil, err - } - - entry := models.NewHashEntry(fileHash, f.Name, fileSize) - if err := hashDoc.AddFile(entry); err != nil { - return nil, err - } - } - - indexReader, err := hashDoc.IndexReader() - if err != nil { - return nil, err - } - if err := blobStorage.Write(hashDoc.Hash, indexReader); err != nil { - return nil, err - } - - tree, err := fs.GetCachedTree(uid) - if err != nil { - return nil, err - } - err = updateTree(tree, blobStorage, func(t *models.HashTree) error { - return tree.Add(hashDoc) - }) - if err != nil { - return nil, err - } - - return &storage.Document{ - ID: docid, - Type: metadata.CollectionType, - Parent: metadata.Parent, - Name: metadata.DocumentName, - }, nil -} - -func createMetadataFile(metadata models.MetadataFile) (r io.Reader, filehash string, size int64, err error) { - jsn, err := json.Marshal(metadata) - if err != nil { - return - } - reader := bytes.NewReader(jsn) - filehash, size, err = models.Hash(reader) - if err != nil { - return - } - reader.Seek(0, io.SeekStart) - r = reader - return -} - // GetBlobURL return a url for a file to store func (fs *FileSystemStorage) GetBlobURL(uid, blobid string, write bool) (docurl string, exp time.Time, err error) { uploadRL := fs.Cfg.StorageURL exp = time.Now().Add(time.Minute * config.ReadStorageExpirationInMinutes) strExp := strconv.FormatInt(exp.Unix(), 10) - scope := ReadScope + scope := storage.ReadScope if write { - scope = WriteScope + scope = storage.WriteScope } - signature, err := SignURLParams([]string{uid, blobid, strExp, scope}, fs.Cfg.JWTSecretKey) + signature, err := storage.SignURLParams([]string{uid, blobid, strExp, scope}, fs.Cfg.JWTSecretKey) if err != nil { return } params := url.Values{ - paramUID: {uid}, - paramBlobID: {blobid}, - paramExp: {strExp}, - paramSignature: {signature}, - paramScope: {scope}, + storage.ParamUID: {uid}, + storage.ParamBlobID: {blobid}, + storage.ParamExp: {strExp}, + storage.ParamSignature: {signature}, + storage.ParamScope: {scope}, } - blobURL := uploadRL + routeBlob + "?" + params.Encode() + blobURL := uploadRL + storage.RouteBlob + "?" + params.Encode() log.Debugln("blobUrl: ", blobURL) return blobURL, exp, nil } // LoadBlob Opens a blob by id -func (fs *FileSystemStorage) LoadBlob(uid, blobid string) (reader io.ReadCloser, gen int64, size int64, hash string, err error) { - generation := int64(0) - blobPath := path.Join(fs.getUserBlobPath(uid), common.Sanitize(blobid)) +func (fs *FileSystemStorage) LoadBlob(uid, blobid string) (reader io.ReadCloser, size int64, hash string, err error) { + uid = common.SanitizeUid(uid) + blobid = common.Sanitize(blobid) + blobPath := path.Join(fs.getUserBlobPath(uid), blobid) log.Debugln("Fullpath:", blobPath) - if blobid == rootBlob { - historyPath := path.Join(fs.getUserBlobPath(uid), historyFile) - lock, err := fslock.Lock(historyPath) - if err != nil { - log.Error("cannot obtain lock") - return nil, 0, 0, "", err - } - defer lock.Unlock() - - fi, err1 := os.Stat(historyPath) - if err1 == nil { - generation = generationFromFileSize(fi.Size()) - } - } fi, err := os.Stat(blobPath) if err != nil || fi.IsDir() { - return nil, generation, 0, "", ErrorNotFound + return nil, 0, "", storage.ErrorNotFound } osFile, err := os.Open(blobPath) @@ -681,78 +72,28 @@ func (fs *FileSystemStorage) LoadBlob(uid, blobid string) (reader io.ReadCloser, return } reader = osFile - return reader, generation, fi.Size(), "crc32c=" + hash, err + return reader, fi.Size(), "crc32c=" + hash, err } // StoreBlob stores a document -func (fs *FileSystemStorage) StoreBlob(uid, id string, stream io.Reader, lastGen int64) (generation int64, err error) { - generation = 1 - - reader := stream - if id == rootBlob { - historyPath := path.Join(fs.getUserBlobPath(uid), historyFile) - var lock fslock.Handle - lock, err = fslock.Lock(historyPath) - if err != nil { - log.Error("cannot obtain lock") - return 0, err - } - defer lock.Unlock() - - currentGen := int64(0) - fi, err1 := os.Stat(historyPath) - if err1 == nil { - currentGen = generationFromFileSize(fi.Size()) - } - - blobPath := path.Join(fs.getUserBlobPath(uid), common.Sanitize(id)) - _, blobErr := os.Stat(blobPath) - rootExists := blobErr == nil - - if currentGen != lastGen && currentGen > 0 && rootExists { - log.Warnf("wrong generation, currentGen %d, lastGen %d", currentGen, lastGen) - return currentGen, ErrorWrongGeneration - } +func (fs *FileSystemStorage) StoreBlob(uid, id string, fileName string, hash string, stream io.Reader) error { + log.Debugf("TODO: check/save etc. write file '%s', hash '%s'", fileName, hash) - var buf bytes.Buffer - tee := io.TeeReader(stream, &buf) - - var hist *os.File - hist, err = os.OpenFile(historyPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) - if err != nil { - return - } - defer hist.Close() - t := time.Now().UTC().Format(time.RFC3339) + " " - hist.WriteString(t) - _, err = io.Copy(hist, tee) - if err != nil { - return - } - hist.WriteString("\n") - - reader = io.NopCloser(&buf) - size, err1 := hist.Seek(0, io.SeekCurrent) - if err1 != nil { - err = err1 - return - } - generation = generationFromFileSize(size) - } - - blobPath := path.Join(fs.getUserBlobPath(uid), common.Sanitize(id)) + uid = common.SanitizeUid(uid) + id = common.Sanitize(id) + blobPath := path.Join(fs.getUserBlobPath(uid), id) log.Info("Write: ", blobPath) file, err := os.Create(blobPath) if err != nil { - return + return err } defer file.Close() - _, err = io.Copy(file, reader) + _, err = io.Copy(file, stream) if err != nil { - return + return err } - return + return nil } // use file size as generation diff --git a/internal/storage/fs/blobstore_test.go b/internal/storage/fs/blobstore_test.go new file mode 100644 index 00000000..68285d8f --- /dev/null +++ b/internal/storage/fs/blobstore_test.go @@ -0,0 +1,150 @@ +package fs + +import ( + "io" + "os" + "path" + "strings" + "testing" + + "github.com/ddvk/rmfakecloud/internal/storage" +) + +func setupBlobDir(t *testing.T, fs *FileSystemStorage, uid string) { + t.Helper() + blobPath := fs.getUserBlobPath(uid) + if err := os.MkdirAll(blobPath, 0700); err != nil { + t.Fatal(err) + } +} + +func TestStoreAndLoadBlob(t *testing.T) { + fs, _ := newTestStorage(t) + uid := "testuser" + setupBlobDir(t, fs, uid) + + blobID := "abc123hash" + content := "blob content data" + + err := fs.StoreBlob(uid, blobID, "test.blob", "", strings.NewReader(content)) + if err != nil { + t.Fatal(err) + } + + reader, size, hash, err := fs.LoadBlob(uid, blobID) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + + if size != int64(len(content)) { + t.Errorf("size mismatch: got %d, want %d", size, len(content)) + } + + if hash == "" { + t.Error("expected non-empty hash") + } + + // Hash should start with "crc32c=" + if !strings.HasPrefix(hash, "crc32c=") { + t.Errorf("hash should start with 'crc32c=', got %q", hash) + } + + data, err := io.ReadAll(reader) + if err != nil { + t.Fatal(err) + } + if string(data) != content { + t.Errorf("content mismatch: got %q, want %q", string(data), content) + } +} + +func TestLoadBlob_NotFound(t *testing.T) { + fs, _ := newTestStorage(t) + uid := "testuser" + setupBlobDir(t, fs, uid) + + _, _, _, err := fs.LoadBlob(uid, "nonexistent") + if err != storage.ErrorNotFound { + t.Fatalf("expected ErrorNotFound, got %v", err) + } +} + +func TestStoreBlob_Overwrite(t *testing.T) { + fs, _ := newTestStorage(t) + uid := "testuser" + setupBlobDir(t, fs, uid) + + blobID := "overwrite-blob" + + err := fs.StoreBlob(uid, blobID, "v1.blob", "", strings.NewReader("version 1")) + if err != nil { + t.Fatal(err) + } + + err = fs.StoreBlob(uid, blobID, "v2.blob", "", strings.NewReader("version 2")) + if err != nil { + t.Fatal(err) + } + + reader, _, _, err := fs.LoadBlob(uid, blobID) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + + data, _ := io.ReadAll(reader) + if string(data) != "version 2" { + t.Errorf("expected version 2, got %q", string(data)) + } +} + +func TestStoreBlob_SanitizesID(t *testing.T) { + fs, _ := newTestStorage(t) + uid := "testuser" + setupBlobDir(t, fs, uid) + + // The ID with path separators should be sanitized + err := fs.StoreBlob(uid, "safe-id", "test", "", strings.NewReader("data")) + if err != nil { + t.Fatal(err) + } + + // Verify the file is in the blob directory with sanitized name + blobPath := path.Join(fs.getUserBlobPath(uid), "safe-id") + if _, err := os.Stat(blobPath); err != nil { + t.Errorf("blob file not found at expected path: %v", err) + } +} + +func TestGetBlobURL(t *testing.T) { + fs, _ := newTestStorage(t) + fs.Cfg.StorageURL = "http://localhost:3000" + fs.Cfg.JWTSecretKey = []byte("test-secret-key-for-signing") + + uid := "testuser" + blobID := "myblobid" + + url, _, err := fs.GetBlobURL(uid, blobID, false) + if err != nil { + t.Fatal(err) + } + if url == "" { + t.Error("expected non-empty URL") + } + if !strings.Contains(url, "/blobstorage") { + t.Errorf("URL should contain /blobstorage, got %q", url) + } + if !strings.Contains(url, "uid="+uid) { + t.Errorf("URL should contain uid param, got %q", url) + } + + // Write URL should contain write scope + writeURL, _, err := fs.GetBlobURL(uid, blobID, true) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(writeURL, "scope=write") { + t.Errorf("write URL should contain write scope, got %q", writeURL) + } +} diff --git a/internal/storage/fs/documentcreator.go b/internal/storage/fs/documentcreator.go index 71a08718..99e3f078 100644 --- a/internal/storage/fs/documentcreator.go +++ b/internal/storage/fs/documentcreator.go @@ -4,7 +4,6 @@ import ( "archive/zip" "encoding/json" "errors" - "fmt" "io" "os" "path" @@ -14,51 +13,11 @@ import ( "github.com/ddvk/rmfakecloud/internal/common" "github.com/ddvk/rmfakecloud/internal/messages" "github.com/ddvk/rmfakecloud/internal/storage" + "github.com/ddvk/rmfakecloud/internal/storage/models" "github.com/google/uuid" "github.com/sirupsen/logrus" ) -func createContent(fileType string) string { - fileType = strings.TrimPrefix(fileType, ".") - str := - ` -{ - "dummyDocument": false, - "extraMetadata": { - "LastPen": "Finelinerv2", - "LastTool": "Finelinerv2", - "ThicknessScale": "", - "LastFinelinerv2Size": "1" - }, - "fileType": "%s", - "fontName": "", - "lastOpenedPage": 0, - "lineHeight": -1, - "margins": 180, - "orientation": "portrait", - "pageCount": 0, - "pages": [], - "textScale": 1, - "transform": { - "m11": 1, - "m12": 0, - "m13": 0, - "m21": 0, - "m22": 1, - "m23": 0, - "m31": 0, - "m32": 0, - "m33": 1 - } -} -` - return fmt.Sprintf(str, fileType) -} - -func extractID(_ io.Reader) (string, error) { - return "", nil -} - func (fs *FileSystemStorage) CreateFolder(uid, name, parent string) (*storage.Document, error) { //create metadata docID := uuid.New().String() @@ -69,7 +28,7 @@ func (fs *FileSystemStorage) CreateFolder(uid, name, parent string) (*storage.Do return nil, err } - metafilePath := fs.getPathFromUser(uid, docID+storage.MetadataFileExt) + metafilePath := fs.getPathFromUser(uid, docID+models.MetadataFileExt) err = os.WriteFile(metafilePath, jsn, 0600) if err != nil { @@ -77,7 +36,7 @@ func (fs *FileSystemStorage) CreateFolder(uid, name, parent string) (*storage.Do } //create zip from pdf - zipfile := fs.getPathFromUser(uid, docID+storage.ZipFileExt) + zipfile := fs.getPathFromUser(uid, docID+models.ZipFileExt) file, err := os.Create(zipfile) if err != nil { return nil, err @@ -87,7 +46,7 @@ func (fs *FileSystemStorage) CreateFolder(uid, name, parent string) (*storage.Do w := zip.NewWriter(file) defer w.Close() - entry, err := w.Create(docID + storage.ContentFileExt) + entry, err := w.Create(docID + models.ContentFileExt) if err != nil { return nil, err } @@ -121,9 +80,9 @@ func (fs *FileSystemStorage) CreateFolder(uid, name, parent string) (*storage.Do func (fs *FileSystemStorage) CreateDocument(uid, filename, parent string, stream io.Reader) (doc *storage.Document, err error) { ext := path.Ext(filename) switch ext { - case storage.PdfFileExt: + case models.PdfFileExt: fallthrough - case storage.EpubFileExt: + case models.EpubFileExt: default: return nil, errors.New("unsupported extension: " + ext) } @@ -131,14 +90,14 @@ func (fs *FileSystemStorage) CreateDocument(uid, filename, parent string, stream var docid string var isZip = false - if ext == storage.ZipFileExt { - docid, err = extractID(stream) + if ext == models.ZipFileExt { + docid, err = models.ExtractID(stream) isZip = true } else { docid = uuid.New().String() } //create zip from pdf - zipfile := fs.getPathFromUser(uid, docid+storage.ZipFileExt) + zipfile := fs.getPathFromUser(uid, docid+models.ZipFileExt) file, err := os.Create(zipfile) if err != nil { return @@ -161,18 +120,18 @@ func (fs *FileSystemStorage) CreateDocument(uid, filename, parent string, stream return } - entry, err = w.Create(docid + storage.PageFileExt) + entry, err = w.Create(docid + models.PageFileExt) if err != nil { return } entry.Write([]byte{}) - entry, err = w.Create(docid + storage.ContentFileExt) + entry, err = w.Create(docid + models.ContentFileExt) if err != nil { return } - content := createContent(ext) + content := models.CreateContent(ext) entry.Write([]byte(content)) } else { logrus.Info("writing file") @@ -198,7 +157,7 @@ func (fs *FileSystemStorage) CreateDocument(uid, filename, parent string, stream Version: 1, } //save metadata - metafilePath := fs.getPathFromUser(uid, docid+storage.MetadataFileExt) + metafilePath := fs.getPathFromUser(uid, docid+models.MetadataFileExt) err = os.WriteFile(metafilePath, jsn, 0600) return } diff --git a/internal/storage/fs/documentcreator_test.go b/internal/storage/fs/documentcreator_test.go index 7c990a2d..46361196 100644 --- a/internal/storage/fs/documentcreator_test.go +++ b/internal/storage/fs/documentcreator_test.go @@ -8,7 +8,7 @@ import ( "testing" "github.com/ddvk/rmfakecloud/internal/config" - "github.com/ddvk/rmfakecloud/internal/storage" + "github.com/ddvk/rmfakecloud/internal/storage/models" ) func TestCreateDocument(t *testing.T) { @@ -32,12 +32,12 @@ func TestCreateDocument(t *testing.T) { t.Error(err) } - _, err = os.Stat(path.Join(userdir, d.ID+storage.MetadataFileExt)) + _, err = os.Stat(path.Join(userdir, d.ID+models.MetadataFileExt)) if err != nil { t.Error(err) } - _, err = os.Stat(path.Join(userdir, d.ID+storage.ZipFileExt)) + _, err = os.Stat(path.Join(userdir, d.ID+models.ZipFileExt)) if err != nil { t.Error(err) } diff --git a/internal/storage/fs/documents.go b/internal/storage/fs/documents.go index f6a4335a..5060df84 100644 --- a/internal/storage/fs/documents.go +++ b/internal/storage/fs/documents.go @@ -17,6 +17,7 @@ import ( "github.com/ddvk/rmfakecloud/internal/config" "github.com/ddvk/rmfakecloud/internal/storage" "github.com/ddvk/rmfakecloud/internal/storage/exporter" + "github.com/ddvk/rmfakecloud/internal/storage/models" ) // DefaultTrashDir name of the trash dir @@ -62,7 +63,7 @@ func (fs *FileSystemStorage) ExportDocument(uid, id, outputType string, exportOp } sanitizedID := common.Sanitize(id) - zipFilePath := fs.getPathFromUser(uid, sanitizedID+storage.ZipFileExt) + zipFilePath := fs.getPathFromUser(uid, sanitizedID+models.ZipFileExt) log.Debugln("Fullpath:", zipFilePath) rawStat, err := os.Stat(zipFilePath) if err != nil { @@ -100,11 +101,13 @@ func (fs *FileSystemStorage) ExportDocument(uid, id, outputType string, exportOp err = exporter.RenderRmapi(arch, outputFile) if err != nil { + outputFile.Close() return nil, err } _, err = outputFile.Seek(0, 0) if err != nil { + outputFile.Close() return nil, err } @@ -114,7 +117,7 @@ func (fs *FileSystemStorage) ExportDocument(uid, id, outputType string, exportOp // GetDocument Opens a document by id func (fs *FileSystemStorage) GetDocument(uid, id string) (io.ReadCloser, error) { - fullPath := fs.getPathFromUser(uid, id+storage.ZipFileExt) + fullPath := fs.getPathFromUser(uid, id+models.ZipFileExt) log.Debugln("Fullpath:", fullPath) reader, err := os.Open(fullPath) return reader, err @@ -130,14 +133,14 @@ func (fs *FileSystemStorage) RemoveDocument(uid, id string) error { } //do not delete, move to trash log.Info(trashDir) - meta := filepath.Base(id + storage.MetadataFileExt) + meta := filepath.Base(id + models.MetadataFileExt) fullPath := fs.getPathFromUser(uid, meta) err = os.Rename(fullPath, path.Join(trashDir, meta)) if err != nil { return err } - zipfile := filepath.Base(id + storage.ZipFileExt) + zipfile := filepath.Base(id + models.ZipFileExt) fullPath = fs.getPathFromUser(uid, zipfile) err = os.Rename(fullPath, path.Join(trashDir, zipfile)) if err != nil { @@ -148,7 +151,7 @@ func (fs *FileSystemStorage) RemoveDocument(uid, id string) error { // StoreDocument stores a document func (fs *FileSystemStorage) StoreDocument(uid, id string, stream io.ReadCloser) error { - fullPath := fs.getPathFromUser(uid, id+storage.ZipFileExt) + fullPath := fs.getPathFromUser(uid, id+models.ZipFileExt) file, err := os.Create(fullPath) if err != nil { return err @@ -164,12 +167,12 @@ func (fs *FileSystemStorage) GetStorageURL(uid, id string) (docurl string, expir exp := time.Now().Add(time.Minute * config.ReadStorageExpirationInMinutes) log.Debugln("uploadUrl: ", uploadRL) - claim := &StorageClaim{ + claim := &storage.StorageClaim{ DocumentID: id, UserID: uid, RegisteredClaims: jwt.RegisteredClaims{ ExpiresAt: jwt.NewNumericDate(exp), - Audience: []string{storageUsage}, + Audience: []string{storage.StorageUsage}, }, } signedToken, err := common.SignClaims(claim, fs.Cfg.JWTSecretKey) @@ -177,5 +180,5 @@ func (fs *FileSystemStorage) GetStorageURL(uid, id string) (docurl string, expir return "", exp, err } - return fmt.Sprintf("%s%s/%s", uploadRL, routeStorage, url.QueryEscape(signedToken)), exp, nil + return fmt.Sprintf("%s%s/%s", uploadRL, storage.RouteStorage, url.QueryEscape(signedToken)), exp, nil } diff --git a/internal/storage/fs/documents_test.go b/internal/storage/fs/documents_test.go new file mode 100644 index 00000000..00dd9b4c --- /dev/null +++ b/internal/storage/fs/documents_test.go @@ -0,0 +1,188 @@ +package fs + +import ( + "io" + "os" + "path" + "strings" + "testing" + + "github.com/ddvk/rmfakecloud/internal/storage/models" +) + +func setupUserDir(t *testing.T, fs *FileSystemStorage, uid string) { + t.Helper() + userPath := fs.getUserPath(uid) + if err := os.MkdirAll(userPath, 0700); err != nil { + t.Fatal(err) + } +} + +func TestStoreAndGetDocument(t *testing.T) { + fs, _ := newTestStorage(t) + uid := "testuser" + setupUserDir(t, fs, uid) + + docID := "test-doc-id" + content := "fake zip content" + + err := fs.StoreDocument(uid, docID, io.NopCloser(strings.NewReader(content))) + if err != nil { + t.Fatal(err) + } + + reader, err := fs.GetDocument(uid, docID) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + + data, err := io.ReadAll(reader) + if err != nil { + t.Fatal(err) + } + + if string(data) != content { + t.Errorf("content mismatch: got %q, want %q", string(data), content) + } +} + +func TestGetDocument_NotFound(t *testing.T) { + fs, _ := newTestStorage(t) + uid := "testuser" + setupUserDir(t, fs, uid) + + _, err := fs.GetDocument(uid, "nonexistent") + if err == nil { + t.Fatal("expected error for nonexistent document") + } +} + +func TestRemoveDocument(t *testing.T) { + fs, dir := newTestStorage(t) + uid := "testuser" + setupUserDir(t, fs, uid) + + docID := "doc-to-remove" + userPath := path.Join(dir, userDir, uid) + + // Create both metadata and zip files + metaPath := path.Join(userPath, docID+models.MetadataFileExt) + zipPath := path.Join(userPath, docID+models.ZipFileExt) + + if err := os.WriteFile(metaPath, []byte(`{"ID":"doc-to-remove"}`), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(zipPath, []byte("zip content"), 0600); err != nil { + t.Fatal(err) + } + + err := fs.RemoveDocument(uid, docID) + if err != nil { + t.Fatal(err) + } + + // Files should be in trash, not in original location + if _, err := os.Stat(metaPath); !os.IsNotExist(err) { + t.Error("metadata file should be moved from original location") + } + if _, err := os.Stat(zipPath); !os.IsNotExist(err) { + t.Error("zip file should be moved from original location") + } + + // Check trash directory + trashDir := path.Join(userPath, DefaultTrashDir) + if _, err := os.Stat(path.Join(trashDir, docID+models.MetadataFileExt)); err != nil { + t.Error("metadata file should exist in trash") + } + if _, err := os.Stat(path.Join(trashDir, docID+models.ZipFileExt)); err != nil { + t.Error("zip file should exist in trash") + } +} + +func TestCreateFolder(t *testing.T) { + fs, dir := newTestStorage(t) + uid := "testuser" + setupUserDir(t, fs, uid) + + doc, err := fs.CreateFolder(uid, "My Folder", "") + if err != nil { + t.Fatal(err) + } + + if doc.Name != "My Folder" { + t.Errorf("name mismatch: got %q", doc.Name) + } + if doc.ID == "" { + t.Error("expected non-empty document ID") + } + + // Check metadata file exists + userPath := path.Join(dir, userDir, uid) + metaPath := path.Join(userPath, doc.ID+models.MetadataFileExt) + if _, err := os.Stat(metaPath); err != nil { + t.Errorf("metadata file not created: %v", err) + } + + // Check zip file exists + zipPath := path.Join(userPath, doc.ID+models.ZipFileExt) + if _, err := os.Stat(zipPath); err != nil { + t.Errorf("zip file not created: %v", err) + } +} + +func TestCreateDocument_PDF(t *testing.T) { + fs, dir := newTestStorage(t) + uid := "testuser" + setupUserDir(t, fs, uid) + + content := strings.NewReader("fake pdf content") + doc, err := fs.CreateDocument(uid, "test.pdf", "", content) + if err != nil { + t.Fatal(err) + } + + if doc.Name != "test" { + t.Errorf("name mismatch: got %q, want %q", doc.Name, "test") + } + if doc.ID == "" { + t.Error("expected non-empty document ID") + } + + // Verify files were created + userPath := path.Join(dir, userDir, uid) + if _, err := os.Stat(path.Join(userPath, doc.ID+models.MetadataFileExt)); err != nil { + t.Error("metadata file not created") + } + if _, err := os.Stat(path.Join(userPath, doc.ID+models.ZipFileExt)); err != nil { + t.Error("zip file not created") + } +} + +func TestCreateDocument_EPUB(t *testing.T) { + fs, _ := newTestStorage(t) + uid := "testuser" + setupUserDir(t, fs, uid) + + content := strings.NewReader("fake epub content") + doc, err := fs.CreateDocument(uid, "book.epub", "", content) + if err != nil { + t.Fatal(err) + } + + if doc.Name != "book" { + t.Errorf("name mismatch: got %q, want %q", doc.Name, "book") + } +} + +func TestCreateDocument_UnsupportedType(t *testing.T) { + fs, _ := newTestStorage(t) + uid := "testuser" + setupUserDir(t, fs, uid) + + content := strings.NewReader("data") + _, err := fs.CreateDocument(uid, "file.docx", "", content) + if err == nil { + t.Fatal("expected error for unsupported file type") + } +} diff --git a/internal/storage/fs/localblobstorage.go b/internal/storage/fs/localblobstorage.go deleted file mode 100644 index 02fcc46d..00000000 --- a/internal/storage/fs/localblobstorage.go +++ /dev/null @@ -1,52 +0,0 @@ -package fs - -import ( - "io" - "strings" - - log "github.com/sirupsen/logrus" -) - -// LocalBlobStorage local file system storage -type LocalBlobStorage struct { - fs *FileSystemStorage - uid string -} - -// GetRootIndex the hash of the root index -func (p *LocalBlobStorage) GetRootIndex() (hash string, gen int64, err error) { - r, gen, _, _, err := p.fs.LoadBlob(p.uid, rootBlob) - if err == ErrorNotFound { - log.Info("root not found") - return "", gen, nil - } - if err != nil { - return "", 0, err - } - defer r.Close() - s, err := io.ReadAll(r) - if err != nil { - return "", 0, err - } - return string(s), int64(gen), nil - -} - -// WriteRootIndex writes the root index -func (p *LocalBlobStorage) WriteRootIndex(generation int64, roothash string) (int64, error) { - r := strings.NewReader(roothash) - return p.fs.StoreBlob(p.uid, rootBlob, r, generation) -} - -// GetReader reader for a given hash -func (p *LocalBlobStorage) GetReader(hash string) (io.ReadCloser, error) { - r, _, _, _, err := p.fs.LoadBlob(p.uid, hash) - return r, err -} - -// Write stores the reader in the hash -func (p *LocalBlobStorage) Write(hash string, r io.Reader) error { - _, err := p.fs.StoreBlob(p.uid, hash, r, -1) - - return err -} diff --git a/internal/storage/fs/metadata.go b/internal/storage/fs/metadata.go index 0c7a2f5e..3651d8fd 100644 --- a/internal/storage/fs/metadata.go +++ b/internal/storage/fs/metadata.go @@ -8,7 +8,7 @@ import ( "strings" "github.com/ddvk/rmfakecloud/internal/messages" - "github.com/ddvk/rmfakecloud/internal/storage" + "github.com/ddvk/rmfakecloud/internal/storage/models" log "github.com/sirupsen/logrus" ) @@ -23,7 +23,7 @@ func (fs *FileSystemStorage) GetAllMetadata(uid string) (result []*messages.RawM for _, f := range files { ext := filepath.Ext(f.Name()) id := strings.TrimSuffix(f.Name(), ext) - if ext != storage.MetadataFileExt { + if ext != models.MetadataFileExt { continue } doc, err := fs.GetMetadata(uid, id) @@ -39,7 +39,7 @@ func (fs *FileSystemStorage) GetAllMetadata(uid string) (result []*messages.RawM // GetMetadata loads a document's metadata func (fs *FileSystemStorage) GetMetadata(uid, id string) (*messages.RawMetadata, error) { - fullPath := fs.getPathFromUser(uid, id+storage.MetadataFileExt) + fullPath := fs.getPathFromUser(uid, id+models.MetadataFileExt) f, err := os.Open(fullPath) if err != nil { return nil, err @@ -62,7 +62,7 @@ func (fs *FileSystemStorage) GetMetadata(uid, id string) (*messages.RawMetadata, // UpdateMetadata updates the metadata of a document func (fs *FileSystemStorage) UpdateMetadata(uid string, r *messages.RawMetadata) error { - filepath := fs.getPathFromUser(uid, r.ID+storage.MetadataFileExt) + filepath := fs.getPathFromUser(uid, r.ID+models.MetadataFileExt) js, err := json.Marshal(r) if err != nil { diff --git a/internal/storage/fs/metadata_test.go b/internal/storage/fs/metadata_test.go new file mode 100644 index 00000000..b45b77b5 --- /dev/null +++ b/internal/storage/fs/metadata_test.go @@ -0,0 +1,127 @@ +package fs + +import ( + "testing" + + "github.com/ddvk/rmfakecloud/internal/common" + "github.com/ddvk/rmfakecloud/internal/messages" +) + +func TestUpdateAndGetMetadata(t *testing.T) { + fs, _ := newTestStorage(t) + uid := "testuser" + setupUserDir(t, fs, uid) + + meta := &messages.RawMetadata{ + ID: "doc-123", + VissibleName: "My Document", + Version: 1, + Type: common.DocumentType, + Parent: "", + } + + if err := fs.UpdateMetadata(uid, meta); err != nil { + t.Fatal(err) + } + + loaded, err := fs.GetMetadata(uid, "doc-123") + if err != nil { + t.Fatal(err) + } + + if loaded.ID != meta.ID { + t.Errorf("ID mismatch: got %q, want %q", loaded.ID, meta.ID) + } + if loaded.VissibleName != meta.VissibleName { + t.Errorf("Name mismatch: got %q, want %q", loaded.VissibleName, meta.VissibleName) + } + if loaded.Type != meta.Type { + t.Errorf("Type mismatch: got %q, want %q", loaded.Type, meta.Type) + } +} + +func TestGetMetadata_NotFound(t *testing.T) { + fs, _ := newTestStorage(t) + uid := "testuser" + setupUserDir(t, fs, uid) + + _, err := fs.GetMetadata(uid, "nonexistent") + if err == nil { + t.Fatal("expected error for nonexistent metadata") + } +} + +func TestGetAllMetadata(t *testing.T) { + fs, _ := newTestStorage(t) + uid := "testuser" + setupUserDir(t, fs, uid) + + docs := []*messages.RawMetadata{ + {ID: "doc-1", VissibleName: "Doc 1", Version: 1, Type: common.DocumentType}, + {ID: "doc-2", VissibleName: "Doc 2", Version: 1, Type: common.DocumentType}, + {ID: "doc-3", VissibleName: "Doc 3", Version: 1, Type: common.CollectionType}, + } + + for _, meta := range docs { + if err := fs.UpdateMetadata(uid, meta); err != nil { + t.Fatal(err) + } + } + + all, err := fs.GetAllMetadata(uid) + if err != nil { + t.Fatal(err) + } + + if len(all) != 3 { + t.Fatalf("expected 3 metadata entries, got %d", len(all)) + } +} + +func TestGetAllMetadata_Empty(t *testing.T) { + fs, _ := newTestStorage(t) + uid := "testuser" + setupUserDir(t, fs, uid) + + all, err := fs.GetAllMetadata(uid) + if err != nil { + t.Fatal(err) + } + + if len(all) != 0 { + t.Fatalf("expected 0 metadata entries, got %d", len(all)) + } +} + +func TestUpdateMetadata_Overwrite(t *testing.T) { + fs, _ := newTestStorage(t) + uid := "testuser" + setupUserDir(t, fs, uid) + + meta := &messages.RawMetadata{ + ID: "doc-overwrite", + VissibleName: "Original", + Version: 1, + Type: common.DocumentType, + } + if err := fs.UpdateMetadata(uid, meta); err != nil { + t.Fatal(err) + } + + meta.VissibleName = "Updated" + meta.Version = 2 + if err := fs.UpdateMetadata(uid, meta); err != nil { + t.Fatal(err) + } + + loaded, err := fs.GetMetadata(uid, "doc-overwrite") + if err != nil { + t.Fatal(err) + } + if loaded.VissibleName != "Updated" { + t.Errorf("expected updated name, got %q", loaded.VissibleName) + } + if loaded.Version != 2 { + t.Errorf("expected version 2, got %d", loaded.Version) + } +} diff --git a/internal/storage/fs/rootstorage.go b/internal/storage/fs/rootstorage.go new file mode 100644 index 00000000..74ea3754 --- /dev/null +++ b/internal/storage/fs/rootstorage.go @@ -0,0 +1,145 @@ +package fs + +import ( + "bufio" + "fmt" + "io" + "os" + "path" + "strings" + "time" + + "github.com/danjacques/gofslock/fslock" + "github.com/ddvk/rmfakecloud/internal/common" + "github.com/ddvk/rmfakecloud/internal/storage" + "github.com/ddvk/rmfakecloud/internal/storage/models" + log "github.com/sirupsen/logrus" +) + +// readRootIndex reads the root hash and generation from the history file without locking. +// Callers must ensure proper locking. +func readRootIndex(historyPath string) (string, int64, error) { + fi, err := os.Stat(historyPath) + if err != nil { + return "", 0, err + } + + if fi.Size() == 0 { + return "", 0, storage.ErrorNotFound + } + + fd, err := os.Open(historyPath) + if err != nil { + return "", 0, err + } + defer fd.Close() + + scanner := bufio.NewScanner(fd) + lastline := "" + var generation int64 + for scanner.Scan() { + line := scanner.Text() + if line != "" { + generation += 1 + lastline = line + } + } + + fields := strings.Fields(lastline) + if len(fields) != 2 { + return "", 0, fmt.Errorf(".root.history corrupted") + } + + return fields[1], generation, nil +} + +func (fs *FileSystemStorage) GetRootIndex(uid string) (string, int64, error) { + uid = common.SanitizeUid(uid) + historyPath := fs.getPathFromUser(uid, historyFile) + + lock, err := fslock.Lock(historyPath) + if err != nil { + log.Error("cannot obtain lock") + return "", 0, err + } + defer lock.Unlock() + + return readRootIndex(historyPath) +} + +func (fs *FileSystemStorage) UpdateRoot(uid string, stream io.Reader, lastGen int64) (int64, error) { + uid = common.SanitizeUid(uid) + historyPath := fs.getPathFromUser(uid, historyFile) + + lock, err := fslock.Lock(historyPath) + if err != nil { + log.Error("cannot obtain lock") + return 0, err + } + defer lock.Unlock() + + currentGen := int64(0) + fi, err := os.Stat(historyPath) + if err == nil { + currentGen = generationFromFileSize(fi.Size()) + } + + rootExists := false + if currentGen > 0 { + rootHash, _, rootErr := readRootIndex(historyPath) + if rootErr == nil && rootHash != "" { + rootHash = common.Sanitize(rootHash) + blobPath := path.Join(fs.getUserBlobPath(uid), rootHash) + _, blobErr := os.Stat(blobPath) + rootExists = blobErr == nil + } + } + + if currentGen != lastGen && currentGen > 0 && rootExists { + log.Warnf("wrong generation, currentGen %d, lastGen %d", currentGen, lastGen) + return currentGen, storage.ErrorWrongGeneration + } + + hist, err := os.OpenFile(historyPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) + if err != nil { + return currentGen, err + } + defer hist.Close() + + hist.WriteString(time.Now().UTC().Format(time.RFC3339) + " ") + _, err = io.Copy(hist, stream) + if err != nil { + return currentGen, err + } + hist.WriteString("\n") + + return currentGen + 1, nil +} + +// GetCachedTree returns the cached blob tree for the user +func (fs *FileSystemStorage) GetCachedTree(uid string, blobStorage models.RemoteStorage) (t *models.HashTree, err error) { + cachePath := path.Join(fs.getUserPath(uid), cachedTreeName) + + tree, err := models.LoadTree(cachePath) + if err != nil { + return nil, err + } + tree.SchemaVersion = fs.Cfg.HashSchemaVersion + changed, err := tree.Mirror(blobStorage) + if err != nil { + return nil, err + } + if changed { + err = tree.Save(cachePath) + if err != nil { + return nil, err + } + } + return tree, nil +} + +// SaveCachedTree saves the cached tree +func (fs *FileSystemStorage) SaveCachedTree(uid string, t *models.HashTree) error { + cachePath := path.Join(fs.getUserPath(uid), cachedTreeName) + return t.Save(cachePath) +} diff --git a/internal/storage/fs/rootstorage_test.go b/internal/storage/fs/rootstorage_test.go new file mode 100644 index 00000000..d71726c7 --- /dev/null +++ b/internal/storage/fs/rootstorage_test.go @@ -0,0 +1,126 @@ +package fs + +import ( + "os" + "path" + "strings" + "testing" + + "github.com/ddvk/rmfakecloud/internal/storage" +) + +func TestGetRootIndex_NoHistory(t *testing.T) { + fs, _ := newTestStorage(t) + uid := "testuser" + setupUserDir(t, fs, uid) + + _, _, err := fs.GetRootIndex(uid) + if err == nil { + t.Fatal("expected error for missing history file") + } +} + +func TestGetRootIndex_EmptyHistory(t *testing.T) { + fs, _ := newTestStorage(t) + uid := "testuser" + setupUserDir(t, fs, uid) + + // Create empty history file + histPath := path.Join(fs.getUserPath(uid), historyFile) + if err := os.WriteFile(histPath, []byte(""), 0600); err != nil { + t.Fatal(err) + } + + _, _, err := fs.GetRootIndex(uid) + if err != storage.ErrorNotFound { + t.Fatalf("expected ErrorNotFound, got %v", err) + } +} + +func TestUpdateAndGetRoot(t *testing.T) { + fs, _ := newTestStorage(t) + uid := "testuser" + setupUserDir(t, fs, uid) + setupBlobDir(t, fs, uid) + + // Use 64-char hashes to match the expected line size (86 = timestamp + space + 64-char hash + newline) + hash1 := "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2" + + // First update (gen 0 -> 1) + gen, err := fs.UpdateRoot(uid, strings.NewReader(hash1), 0) + if err != nil { + t.Fatal(err) + } + if gen != 1 { + t.Errorf("expected generation 1, got %d", gen) + } + + // Read back + rootHash, rootGen, err := fs.GetRootIndex(uid) + if err != nil { + t.Fatal(err) + } + if rootHash != hash1 { + t.Errorf("hash mismatch: got %q, want %q", rootHash, hash1) + } + if rootGen != 1 { + t.Errorf("generation mismatch: got %d, want 1", rootGen) + } + + // Create root blob so the generation check recognizes existing root + blobPath := path.Join(fs.getUserBlobPath(uid), hash1) + if err := os.WriteFile(blobPath, []byte("blob"), 0600); err != nil { + t.Fatal(err) + } + + // Second update (gen 1 -> 2) + hash2 := "f1e2d3c4b5a6f1e2d3c4b5a6f1e2d3c4b5a6f1e2d3c4b5a6f1e2d3c4b5a6f1e2" + gen, err = fs.UpdateRoot(uid, strings.NewReader(hash2), 1) + if err != nil { + t.Fatal(err) + } + if gen != 2 { + t.Errorf("expected generation 2, got %d", gen) + } +} + +func TestUpdateRoot_WrongGeneration(t *testing.T) { + fs, _ := newTestStorage(t) + uid := "testuser" + setupUserDir(t, fs, uid) + setupBlobDir(t, fs, uid) + + // Use 64-char hash for correct generation calculation + hash1 := "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2" + _, err := fs.UpdateRoot(uid, strings.NewReader(hash1), 0) + if err != nil { + t.Fatal(err) + } + + // Create root blob so generation check recognizes the root exists + blobPath := path.Join(fs.getUserBlobPath(uid), hash1) + if err := os.WriteFile(blobPath, []byte("blob"), 0600); err != nil { + t.Fatal(err) + } + + // Try to update with wrong generation (99 != 1) + hash2 := "f1e2d3c4b5a6f1e2d3c4b5a6f1e2d3c4b5a6f1e2d3c4b5a6f1e2d3c4b5a6f1e2" + _, err = fs.UpdateRoot(uid, strings.NewReader(hash2), 99) + if err != storage.ErrorWrongGeneration { + t.Fatalf("expected ErrorWrongGeneration, got %v", err) + } +} + +func TestReadRootIndex_CorruptedHistory(t *testing.T) { + // Create a history file with bad format + dir := t.TempDir() + histPath := path.Join(dir, "corrupted.history") + if err := os.WriteFile(histPath, []byte("badline\n"), 0600); err != nil { + t.Fatal(err) + } + + _, _, err := readRootIndex(histPath) + if err == nil { + t.Fatal("expected error for corrupted history") + } +} diff --git a/internal/storage/fs/userstorage.go b/internal/storage/fs/userstorage.go index 20b2fe2d..9ad62406 100644 --- a/internal/storage/fs/userstorage.go +++ b/internal/storage/fs/userstorage.go @@ -8,12 +8,17 @@ import ( "github.com/ddvk/rmfakecloud/internal/config" "github.com/ddvk/rmfakecloud/internal/model" + log "github.com/sirupsen/logrus" ) const ( - userDir = "users" - profileName = ".userprofile" + cachedTreeName = ".tree" + profileName = ".userprofile" + userDir = "users" + + // serves as root modification log and generation number source + historyFile = ".root.history" ) // NewStorage new file system storage diff --git a/internal/storage/fs/userstorage_test.go b/internal/storage/fs/userstorage_test.go new file mode 100644 index 00000000..5020e39e --- /dev/null +++ b/internal/storage/fs/userstorage_test.go @@ -0,0 +1,220 @@ +package fs + +import ( + "os" + "path" + "testing" + + "github.com/ddvk/rmfakecloud/internal/config" + "github.com/ddvk/rmfakecloud/internal/model" +) + +func newTestStorage(t *testing.T) (*FileSystemStorage, string) { + t.Helper() + dir := t.TempDir() + cfg := &config.Config{ + DataDir: dir, + } + return NewStorage(cfg), dir +} + +func TestRegisterUser(t *testing.T) { + fs, dir := newTestStorage(t) + + user, err := model.NewUser("testuser@example.com", "password123") + if err != nil { + t.Fatal(err) + } + + err = fs.RegisterUser(user) + if err != nil { + t.Fatal(err) + } + + // Profile file should exist + profilePath := path.Join(dir, userDir, user.ID, profileName) + if _, err := os.Stat(profilePath); err != nil { + t.Fatalf("profile file not created: %v", err) + } + + // Sync directory should exist + syncPath := path.Join(dir, userDir, user.ID, SyncFolder) + if _, err := os.Stat(syncPath); err != nil { + t.Fatalf("sync directory not created: %v", err) + } +} + +func TestRegisterUser_EmptyID(t *testing.T) { + fs, _ := newTestStorage(t) + + user := &model.User{ID: ""} + err := fs.RegisterUser(user) + if err == nil { + t.Fatal("expected error for empty id") + } +} + +func TestRegisterUser_Duplicate(t *testing.T) { + fs, _ := newTestStorage(t) + + user, err := model.NewUser("dup@example.com", "pass") + if err != nil { + t.Fatal(err) + } + + if err := fs.RegisterUser(user); err != nil { + t.Fatal(err) + } + + // Second registration should fail (O_EXCL) + err = fs.RegisterUser(user) + if err == nil { + t.Fatal("expected error for duplicate registration") + } +} + +func TestGetUser(t *testing.T) { + fs, _ := newTestStorage(t) + + original, err := model.NewUser("getuser@example.com", "pass123") + if err != nil { + t.Fatal(err) + } + original.Name = "Test User" + original.Nickname = "testy" + + if err := fs.RegisterUser(original); err != nil { + t.Fatal(err) + } + + loaded, err := fs.GetUser(original.ID) + if err != nil { + t.Fatal(err) + } + + if loaded.ID != original.ID { + t.Errorf("ID mismatch: got %q, want %q", loaded.ID, original.ID) + } + if loaded.Email != original.Email { + t.Errorf("Email mismatch: got %q, want %q", loaded.Email, original.Email) + } + if loaded.Name != original.Name { + t.Errorf("Name mismatch: got %q, want %q", loaded.Name, original.Name) + } +} + +func TestGetUser_Empty(t *testing.T) { + fs, _ := newTestStorage(t) + + _, err := fs.GetUser("") + if err == nil { + t.Fatal("expected error for empty user") + } +} + +func TestGetUser_NotFound(t *testing.T) { + fs, _ := newTestStorage(t) + + _, err := fs.GetUser("nonexistent") + if err == nil { + t.Fatal("expected error for nonexistent user") + } +} + +func TestGetUsers(t *testing.T) { + fs, _ := newTestStorage(t) + + for _, email := range []string{"user1@test.com", "user2@test.com", "user3@test.com"} { + u, err := model.NewUser(email, "pass") + if err != nil { + t.Fatal(err) + } + if err := fs.RegisterUser(u); err != nil { + t.Fatal(err) + } + } + + users, err := fs.GetUsers() + if err != nil { + t.Fatal(err) + } + + if len(users) != 3 { + t.Fatalf("expected 3 users, got %d", len(users)) + } +} + +func TestUpdateUser(t *testing.T) { + fs, _ := newTestStorage(t) + + user, err := model.NewUser("update@test.com", "pass") + if err != nil { + t.Fatal(err) + } + + if err := fs.RegisterUser(user); err != nil { + t.Fatal(err) + } + + user.Name = "Updated Name" + user.Nickname = "updated" + + if err := fs.UpdateUser(user); err != nil { + t.Fatal(err) + } + + loaded, err := fs.GetUser(user.ID) + if err != nil { + t.Fatal(err) + } + if loaded.Name != "Updated Name" { + t.Errorf("Name not updated: got %q", loaded.Name) + } + if loaded.Nickname != "updated" { + t.Errorf("Nickname not updated: got %q", loaded.Nickname) + } +} + +func TestUpdateUser_EmptyID(t *testing.T) { + fs, _ := newTestStorage(t) + + err := fs.UpdateUser(&model.User{ID: ""}) + if err == nil { + t.Fatal("expected error for empty id") + } +} + +func TestRemoveUser(t *testing.T) { + fs, dir := newTestStorage(t) + + user, err := model.NewUser("remove@test.com", "pass") + if err != nil { + t.Fatal(err) + } + + if err := fs.RegisterUser(user); err != nil { + t.Fatal(err) + } + + userPath := path.Join(dir, userDir, user.ID) + if _, err := os.Stat(userPath); err != nil { + t.Fatal("user dir should exist before removal") + } + + if err := fs.RemoveUser(user.ID); err != nil { + t.Fatal(err) + } + + if _, err := os.Stat(userPath); !os.IsNotExist(err) { + t.Fatal("user dir should be removed") + } +} + +func TestRemoveUser_EmptyID(t *testing.T) { + fs, _ := newTestStorage(t) + + err := fs.RemoveUser("") + if err == nil { + t.Fatal("expected error for empty id") + } +} diff --git a/internal/storage/models/archive.go b/internal/storage/models/archive.go index d244db6d..1580a6d4 100644 --- a/internal/storage/models/archive.go +++ b/internal/storage/models/archive.go @@ -3,10 +3,10 @@ package models import ( "encoding/json" "io" + "io/ioutil" "path" "strings" - "github.com/ddvk/rmfakecloud/internal/storage" "github.com/ddvk/rmfakecloud/internal/storage/exporter" "github.com/juruen/rmapi/archive" "github.com/juruen/rmapi/encoding/rm" @@ -27,7 +27,7 @@ func ArchiveFromHashDoc(doc *HashDoc, rs RemoteStorage) (*exporter.MyArchive, er filext := path.Ext(f.EntryName) name := strings.TrimSuffix(path.Base(f.EntryName), filext) switch filext { - case storage.ContentFileExt: + case ContentFileExt: blob, err := rs.GetReader(f.Hash) if err != nil { return nil, err @@ -41,25 +41,27 @@ func ArchiveFromHashDoc(doc *HashDoc, rs RemoteStorage) (*exporter.MyArchive, er if err != nil { return nil, err } - case storage.EpubFileExt: + case EpubFileExt: fallthrough - case storage.PdfFileExt: + case PdfFileExt: blob, err := rs.GetReader(f.Hash) if err != nil { return nil, err } - // defer blob.Close() - // contentBytes, err := ioutil.ReadAll(blob) - // if err != nil { - // return nil, err - // } - // a.Payload = contentBytes - //HACK: - a.PayloadReader = blob.(io.ReadSeekCloser) + if rsc, ok := blob.(io.ReadSeekCloser); ok { + a.PayloadReader = rsc + } else { + defer blob.Close() + contentBytes, err := ioutil.ReadAll(blob) + if err != nil { + return nil, err + } + a.Payload = contentBytes + } case ".json": //metadata - case storage.RmFileExt: + case RmFileExt: log.Debug("adding page ", name) pageMap[name] = f.Hash } diff --git a/internal/storage/models/documentcreator.go b/internal/storage/models/documentcreator.go new file mode 100644 index 00000000..e1d34b26 --- /dev/null +++ b/internal/storage/models/documentcreator.go @@ -0,0 +1,48 @@ +package models + +import ( + "fmt" + "io" + "strings" +) + +func CreateContent(fileType string) string { + fileType = strings.TrimPrefix(fileType, ".") + str := + ` +{ + "dummyDocument": false, + "extraMetadata": { + "LastPen": "Finelinerv2", + "LastTool": "Finelinerv2", + "ThicknessScale": "", + "LastFinelinerv2Size": "1" + }, + "fileType": "%s", + "fontName": "", + "lastOpenedPage": 0, + "lineHeight": -1, + "margins": 180, + "orientation": "portrait", + "pageCount": 0, + "pages": [], + "textScale": 1, + "transform": { + "m11": 1, + "m12": 0, + "m13": 0, + "m21": 0, + "m22": 1, + "m23": 0, + "m31": 0, + "m32": 0, + "m33": 1 + } +} +` + return fmt.Sprintf(str, fileType) +} + +func ExtractID(_ io.Reader) (string, error) { + return "", nil +} diff --git a/internal/storage/filetypes.go b/internal/storage/models/filetypes.go similarity index 94% rename from internal/storage/filetypes.go rename to internal/storage/models/filetypes.go index e1bf49e7..b0337dee 100644 --- a/internal/storage/filetypes.go +++ b/internal/storage/models/filetypes.go @@ -1,4 +1,4 @@ -package storage +package models const ( MetadataFileExt = ".metadata" diff --git a/internal/storage/models/hashdoc.go b/internal/storage/models/hashdoc.go index b46a9cf0..d6692a4b 100644 --- a/internal/storage/models/hashdoc.go +++ b/internal/storage/models/hashdoc.go @@ -60,7 +60,7 @@ func (d *HashDoc) Rehash() error { return nil } -func (d *HashDoc) MetadataReader() (hash string, reader io.Reader, err error) { +func (d *HashDoc) MetadataReader() (hash string, crc32c string, reader io.Reader, err error) { jsn, err := json.Marshal(d.MetadataFile) if err != nil { return @@ -69,6 +69,10 @@ func (d *HashDoc) MetadataReader() (hash string, reader io.Reader, err error) { sha.Write(jsn) hash = hex.EncodeToString(sha.Sum(nil)) log.Info("new hash: ", hash) + crc32c, err = common.CRC32CFromReader(bytes.NewReader(jsn)) + if err != nil { + return + } reader = bytes.NewReader(jsn) found := false for _, f := range d.Files { diff --git a/internal/storage/models/hashentry.go b/internal/storage/models/hashentry.go index d473e529..3f2ea948 100644 --- a/internal/storage/models/hashentry.go +++ b/internal/storage/models/hashentry.go @@ -3,8 +3,6 @@ package models import ( "strconv" "strings" - - "github.com/ddvk/rmfakecloud/internal/storage" ) // NewHashEntry blah @@ -28,10 +26,10 @@ type HashEntry struct { // IsMetadata if this entry points to a metadata blob func (h *HashEntry) IsMetadata() bool { - return strings.HasSuffix(h.EntryName, storage.MetadataFileExt) + return strings.HasSuffix(h.EntryName, MetadataFileExt) } func (h *HashEntry) IsContent() bool { - return strings.HasSuffix(h.EntryName, storage.ContentFileExt) + return strings.HasSuffix(h.EntryName, ContentFileExt) } // Line a line in the index file diff --git a/internal/storage/models/hashtree.go b/internal/storage/models/hashtree.go index b618bad5..6a466325 100644 --- a/internal/storage/models/hashtree.go +++ b/internal/storage/models/hashtree.go @@ -11,6 +11,7 @@ import ( "sort" "strconv" + "github.com/ddvk/rmfakecloud/internal/common" log "github.com/sirupsen/logrus" ) @@ -37,15 +38,17 @@ func HashEntries(entries []*HashEntry) (string, error) { return hashStr, nil } -func Hash(r io.Reader) (string, int64, error) { +func Hash(r io.Reader) (string, string, int64, error) { hasher := sha256.New() - w, err := io.Copy(hasher, r) + crc32c := common.CRC32CWriter() + mw := io.MultiWriter(hasher, crc32c) + w, err := io.Copy(mw, r) if err != nil { - return "", w, err + return "", "", w, err } h := hasher.Sum(nil) hstr := hex.EncodeToString(h) - return hstr, w, err + return hstr, common.CRC32CSum(crc32c), w, err } func FileHashAndSize(file string) ([]byte, int64, error) { f, err := os.Open(file) diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 21f29dd7..df90ff16 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -7,6 +7,7 @@ import ( "github.com/ddvk/rmfakecloud/internal/common" "github.com/ddvk/rmfakecloud/internal/messages" "github.com/ddvk/rmfakecloud/internal/model" + "github.com/ddvk/rmfakecloud/internal/storage/models" ) // ExportOption type of export @@ -27,15 +28,15 @@ type DocumentStorer interface { GetStorageURL(uid, docid string) (string, time.Time, error) CreateDocument(uid, name, parent string, stream io.Reader) (doc *Document, err error) + CreateFolder(uid, name, parent string) (doc *Document, err error) } // BlobStorage stuff for sync15 type BlobStorage interface { GetBlobURL(uid, docid string, write bool) (string, time.Time, error) - StoreBlob(uid, blobID string, s io.Reader, matchGeneration int64) (int64, error) - LoadBlob(uid, blobID string) (reader io.ReadCloser, gen int64, size int64, crc32c string, err error) - CreateBlobDocument(uid, name, parent string, stream io.Reader) (doc *Document, err error) + StoreBlob(uid, blobID string, filename string, hash string, s io.Reader) error + LoadBlob(uid, blobID string) (reader io.ReadCloser, size int64, crc32c string, err error) } // MetadataStorer manages document metadata @@ -45,6 +46,14 @@ type MetadataStorer interface { GetMetadata(uid, docid string) (*messages.RawMetadata, error) } +// RootStorer holds informations about users +type RootStorer interface { + GetCachedTree(uid string, rs models.RemoteStorage) (t *models.HashTree, err error) + GetRootIndex(uid string) (string, int64, error) + SaveCachedTree(uid string, t *models.HashTree) error + UpdateRoot(uid string, stream io.Reader, lastGen int64) (int64, error) +} + // UserStorer holds informations about users type UserStorer interface { GetUsers() ([]*model.User, error) @@ -54,6 +63,13 @@ type UserStorer interface { RemoveUser(uid string) error } +type UserRootStorer interface { + GetCachedTree() (t *models.HashTree, err error) + GetRootIndex() (string, int64, error) + SaveCachedTree(t *models.HashTree) error + UpdateRoot(stream io.Reader, lastGen int64) (int64, error) +} + // Document represents a document in storage type Document struct { ID string diff --git a/internal/ui/handlers.go b/internal/ui/handlers.go index 3cc82b49..34e89bdb 100644 --- a/internal/ui/handlers.go +++ b/internal/ui/handlers.go @@ -32,9 +32,7 @@ const ( ) func userID(c *gin.Context) string { - //TODO: suppress the warning - //codeql[go/path-injection] - return c.GetString(userIDContextKey) + return common.SanitizeUid(c.GetString(userIDContextKey)) } func (app *ReactAppWrapper) register(c *gin.Context) {