From 5ecd9a6e97378c0480555415356c70ffaf5988b5 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Wed, 31 May 2023 14:30:13 +0200 Subject: [PATCH 1/6] ui: Add interface to manage integrations --- internal/ui/handlers.go | 139 ++++++++++++++ internal/ui/routes.go | 7 + ui/src/App.jsx | 2 + ui/src/components/Navigation.js | 5 + ui/src/pages/Integrations/IntegrationModal.js | 172 ++++++++++++++++++ .../pages/Integrations/NewIntegrationModal.js | 153 ++++++++++++++++ ui/src/pages/Integrations/index.jsx | 116 ++++++++++++ ui/src/services/api.service.js | 30 +++ 8 files changed, 624 insertions(+) create mode 100644 ui/src/pages/Integrations/IntegrationModal.js create mode 100644 ui/src/pages/Integrations/NewIntegrationModal.js create mode 100644 ui/src/pages/Integrations/index.jsx diff --git a/internal/ui/handlers.go b/internal/ui/handlers.go index 7a32f406..e30e7823 100644 --- a/internal/ui/handlers.go +++ b/internal/ui/handlers.go @@ -20,6 +20,7 @@ const ( browserIDContextKey = "browserID" isSync15Key = "sync15" docIDParam = "docid" + intIDParam = "intid" uiLogger = "[ui] " ui10 = " [10] " useridParam = "userid" @@ -509,3 +510,141 @@ func (app *ReactAppWrapper) createUser(c *gin.Context) { } c.Status(http.StatusCreated) } + +func (app *ReactAppWrapper) listIntegrations(c *gin.Context) { + uid := c.GetString(userIDContextKey) + + user, err := app.userStorer.GetUser(uid) + if err != nil { + log.Error(err) + c.AbortWithStatus(http.StatusUnauthorized) + return + } + + c.JSON(http.StatusOK, user.Integrations) +} + +func (app *ReactAppWrapper) createIntegration(c *gin.Context) { + int := model.IntegrationConfig{} + if err := c.ShouldBindJSON(&int); err != nil { + log.Error(err) + badReq(c, err.Error()) + return + } + + uid := c.GetString(userIDContextKey) + + user, err := app.userStorer.GetUser(uid) + if err != nil { + log.Error(err) + c.AbortWithStatus(http.StatusUnauthorized) + return + } + + int.ID = uuid.NewString() + user.Integrations = append(user.Integrations, int) + + err = app.userStorer.UpdateUser(user) + + if err != nil { + log.Error("error updating user", err) + c.AbortWithStatus(http.StatusInternalServerError) + return + } + + c.JSON(http.StatusOK, int) +} + +func (app *ReactAppWrapper) getIntegration(c *gin.Context) { + uid := c.GetString(userIDContextKey) + + intid := common.ParamS(intIDParam, c) + + user, err := app.userStorer.GetUser(uid) + if err != nil { + log.Error(err) + c.AbortWithStatus(http.StatusUnauthorized) + return + } + + for _, integration := range user.Integrations { + if integration.ID == intid { + c.JSON(http.StatusOK, integration) + return + } + } + + c.AbortWithStatus(http.StatusNotFound) +} + +func (app *ReactAppWrapper) updateIntegration(c *gin.Context) { + int := model.IntegrationConfig{} + if err := c.ShouldBindJSON(&int); err != nil { + log.Error(err) + badReq(c, err.Error()) + return + } + + uid := c.GetString(userIDContextKey) + + intid := common.ParamS(intIDParam, c) + + user, err := app.userStorer.GetUser(uid) + if err != nil { + log.Error(err) + c.AbortWithStatus(http.StatusUnauthorized) + return + } + + for idx, integration := range user.Integrations { + if integration.ID == intid { + int.ID = integration.ID + user.Integrations[idx] = int + + err = app.userStorer.UpdateUser(user) + + if err != nil { + log.Error("error updating user", err) + c.AbortWithStatus(http.StatusInternalServerError) + return + } + + c.JSON(http.StatusOK, int) + return + } + } + + c.AbortWithStatus(http.StatusNotFound) +} + +func (app *ReactAppWrapper) deleteIntegration(c *gin.Context) { + uid := c.GetString(userIDContextKey) + + intid := common.ParamS(intIDParam, c) + + user, err := app.userStorer.GetUser(uid) + if err != nil { + log.Error(err) + c.AbortWithStatus(http.StatusUnauthorized) + return + } + + for idx, integration := range user.Integrations { + if integration.ID == intid { + user.Integrations = append(user.Integrations[:idx], user.Integrations[idx+1:]...) + + err = app.userStorer.UpdateUser(user) + + if err != nil { + log.Error("error updating user", err) + c.AbortWithStatus(http.StatusInternalServerError) + return + } + + c.Status(http.StatusAccepted) + return + } + } + + c.AbortWithStatus(http.StatusNotFound) +} diff --git a/internal/ui/routes.go b/internal/ui/routes.go index 718efaaa..4df7fd14 100644 --- a/internal/ui/routes.go +++ b/internal/ui/routes.go @@ -72,6 +72,13 @@ func (app *ReactAppWrapper) RegisterRoutes(router *gin.Engine) { auth.POST("folders", app.createFolder) auth.GET("documents/:docid/metadata", app.getDocumentMetadata) + // integrations + auth.GET("integrations", app.listIntegrations) + auth.POST("integrations", app.createIntegration) + auth.GET("integrations/:intid", app.getIntegration) + auth.PUT("integrations/:intid", app.updateIntegration) + auth.DELETE("integrations/:intid", app.deleteIntegration) + //admin admin := auth.Group("") admin.Use(app.adminMiddleware()) diff --git a/ui/src/App.jsx b/ui/src/App.jsx index 67fe8997..e764c669 100644 --- a/ui/src/App.jsx +++ b/ui/src/App.jsx @@ -12,6 +12,7 @@ import Login from "./pages/Login"; import Home from "./pages/Home"; import Connect from "./pages/Connect"; import Documents from "./pages/Documents"; +import Integrations from "./pages/Integrations"; import Profile from "./pages/Profile"; import Admin from "./pages/Admin"; import NoMatch from "./pages/404"; @@ -35,6 +36,7 @@ export default function App() { + diff --git a/ui/src/components/Navigation.js b/ui/src/components/Navigation.js index cc1c4aa4..4607efc5 100644 --- a/ui/src/components/Navigation.js +++ b/ui/src/components/Navigation.js @@ -33,6 +33,11 @@ const NavigationBar = () => { Documents + + + Integrations + + Connect diff --git a/ui/src/pages/Integrations/IntegrationModal.js b/ui/src/pages/Integrations/IntegrationModal.js new file mode 100644 index 00000000..a2f9d156 --- /dev/null +++ b/ui/src/pages/Integrations/IntegrationModal.js @@ -0,0 +1,172 @@ +import React, { useState } from "react"; +import Form from "react-bootstrap/Form"; +import { Button, Card } from "react-bootstrap"; +import apiService from "../../services/api.service"; + +import { Alert } from "react-bootstrap"; + +export default function IntegrationModal(params) { + const { integration, onSave, headerText, onClose } = params; + + const [formErrors, setFormErrors] = useState({}); + const [integrationForm, setIntegrationForm] = useState({ + name: integration?.Name, + provider: integration?.Provider, + email: integration?.email, + username: integration?.Username, + password: integration?.Password, + address: integration?.Address, + insecure: integration?.Insecure, + accesstoken: integration?.Accesstoken, + path: integration?.Path, + }); + + function handleChange({ target }) { + setIntegrationForm({ ...integrationForm, [target.name]: target.value }); + } + + function formIsValid() { + const _errors = {}; + + if (!integrationForm.name) _errors.error = "name is required"; + + setFormErrors(_errors); + + return Object.keys(_errors).length === 0; + } + + async function handleSubmit(event) { + event.preventDefault(); + + if (!formIsValid()) return; + + try { + await apiService.updateintegration({ + id: integration.ID, + name: integrationForm.name, + provider: integrationForm.provider, + username: integrationForm.username, + password: integrationForm.password, + address: integrationForm.address, + insecure: integrationForm.insecure, + accesstoken: integrationForm.accesstoken, + path: integrationForm.path, + }); + onSave(); + } catch (e) { + setFormErrors({ error: e.toString() }); + } + } + + if (!integration) return null; + return ( +
+ + + {headerText} + + +
+ + + IntegrationID + + + Provider + + + + + + + Name + + + {integrationForm.provider === "webdav" && ( + <> + Address + + + )} + {integrationForm.provider === "webdav" && ( + <> + Username + + + )} + {integrationForm.provider === "webdav" && ( + <> + Password + + + )} + + {integrationForm.provider === "localfs" && ( + <> + Path + + + )} + + {integrationForm.provider === "dropbox" && ( + <> + Access Token + + + )} +
+ + + + + + +
+ ); +} diff --git a/ui/src/pages/Integrations/NewIntegrationModal.js b/ui/src/pages/Integrations/NewIntegrationModal.js new file mode 100644 index 00000000..7490f37b --- /dev/null +++ b/ui/src/pages/Integrations/NewIntegrationModal.js @@ -0,0 +1,153 @@ +import React, { useState } from "react"; +import Form from "react-bootstrap/Form"; +import { Button, Card } from "react-bootstrap"; +import apiService from "../../services/api.service"; + +import { Alert } from "react-bootstrap"; + +export default function IntegrationProfileModal(params) { + const { onSave, onClose } = params; + + const [formErrors, setFormErrors] = useState({}); + const [formInfo, setFormInfo] = useState({}); + const [integrationForm, setIntegrationForm] = useState({ + name: "", + provider: "localfs", + }); + + function handleChange({ target }) { + setIntegrationForm({ ...integrationForm, [target.name]: target.value }); + } + + function formIsValid() { + const _errors = {}; + + if (!integrationForm.name) _errors.error = "name is required"; + + if (!integrationForm.provider) _errors.error = "provider is required"; + + setFormErrors(_errors); + + return Object.keys(_errors).length === 0; + } + + async function handleSubmit(event) { + event.preventDefault(); + + if (!formIsValid()) return; + + try { + await apiService.createintegration(integrationForm); + setFormInfo({ message: "Created" }); + onSave(); + } catch (e) { + setFormErrors({ error: e.toString() }); + } + } + + return ( +
+ + + New Integration + + + + + + + Name + + + Provider + + + + + + + {integrationForm.provider === "webdav" && ( + <> + Address + + + )} + {integrationForm.provider === "webdav" && ( + <> + Username + + + )} + {integrationForm.provider === "webdav" && ( + <> + Password + + + )} + + {integrationForm.provider === "localfs" && ( + <> + Path + + + )} + + {integrationForm.provider === "dropbox" && ( + <> + Access Token + + + )} + + + + + + +
+ ); +} diff --git a/ui/src/pages/Integrations/index.jsx b/ui/src/pages/Integrations/index.jsx new file mode 100644 index 00000000..1c94cc75 --- /dev/null +++ b/ui/src/pages/Integrations/index.jsx @@ -0,0 +1,116 @@ +import React, {useState} from "react"; +import useFetch from "../../hooks/useFetch"; +import Spinner from "../../components/Spinner"; +import {Alert, Button, Card, Container, Modal, Table} from "react-bootstrap"; +import IntegrationModal from "./IntegrationModal"; +import NewIntegrationModal from "./NewIntegrationModal"; +import apiService from "../../services/api.service"; +import { toast } from "react-toastify"; +const integrationListUrl = "integrations"; + +const NewIntegration = 1; +const UpdateIntegration = 2; +const Integrations = () => { + const [index, setIndex] = useState(0); + const { data: integrationList, error, loading } = useFetch(`${integrationListUrl}`, index); + const [ state, setState ] = useState({showModal: 0, modalIntegration: null}); + const refresh = () =>{ + setIndex(previous => previous+1) + } + + function openModal(index: number) { + if (!integrationList) return; + let integration = integrationList[index]; + setState({ + showModal: UpdateIntegration, + modalIntegration: integration, + }); + } + function closeModal() { + setState({ + showModal: 0, + modalIntegration: null, + }); + } + + if (loading) { + return + } + + if (error) { + return ( + + An Error Occurred + {`Error ${error.status}: ${error.statusText}`} + + ); + } + + const newIntegration = e => { + setState({ + showModal: NewIntegration, + }); + } + + const onSave = () => { + closeModal(); + refresh(); + } + + const remove = async (e, id, name) => { + e.preventDefault() + e.stopPropagation() + if (!window.confirm(`Are you sure you want to delete integration: ${name}?`)) + return false + + try{ + await apiService.deleteintegration(id) + refresh() + } catch(e){ + toast.error('Error:'+ e) + } + } + + return ( + +

Integrations

+ + + + + + + + + + + + + {!integrationList.length && ( + + + + )} + {integrationList.map((i, index) => ( + openModal(index)} style={{ cursor: "pointer" }}> + + + + + + + ))} + +
#IntegrationIdNameProvider
No integration
{index}{i.ID}{i.Name}{i.Provider}
+ + + + + + + + + ); +}; + +export default Integrations; diff --git a/ui/src/services/api.service.js b/ui/src/services/api.service.js index 7fde7925..a8a07ef5 100644 --- a/ui/src/services/api.service.js +++ b/ui/src/services/api.service.js @@ -127,6 +127,36 @@ class ApiServices { headers: this.header(), }).then((r) => handleError(r)); } + + listintegration() { + return fetch(`${constants.ROOT_URL}/integrations`, { + method: "GET", + headers: this.header(), + }).then((r) => { + handleError(r); + return r.json(); + }); + } + updateintegration(integration) { + return fetch(`${constants.ROOT_URL}/integrations/${integration.id}`, { + method: "PUT", + headers: this.header(), + body: JSON.stringify(integration), + }).then((r) => handleError(r)); + } + createintegration(integration) { + return fetch(`${constants.ROOT_URL}/integrations`, { + method: "POST", + headers: this.header(), + body: JSON.stringify(integration), + }).then((r) => handleError(r)); + } + deleteintegration(integrationid) { + return fetch(`${constants.ROOT_URL}/integrations/${integrationid}`, { + method: "DELETE", + headers: this.header(), + }).then((r) => handleError(r)); + } } function removeUser(){ From 2937e8eacb102435873fcc3cd45672b92628de39 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Wed, 31 May 2023 14:34:15 +0200 Subject: [PATCH 2/6] Reorder providers --- internal/integrations/integrations.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/integrations/integrations.go b/internal/integrations/integrations.go index a7f4986c..b4f59dc5 100644 --- a/internal/integrations/integrations.go +++ b/internal/integrations/integrations.go @@ -37,12 +37,12 @@ func GetIntegrationProvider(storer storage.UserStorer, uid, integrationid string continue } switch intg.Provider { - case webdavProvider: - return newWebDav(intg), nil case dropboxProvider: return newDropbox(intg), nil case localfsProvider: return newLocalFS(intg), nil + case webdavProvider: + return newWebDav(intg), nil } } return nil, fmt.Errorf("integration not found or no implmentation (only webdav) %s", integrationid) From 1eaa6a06cb82e9a80bb53821dbab6cf05c42f0a5 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sun, 17 Nov 2024 15:03:56 +0100 Subject: [PATCH 3/6] Implement metadata retrieval for integrations --- internal/app/handlers.go | 22 +++++++++++++++++++--- internal/integrations/dropbox.go | 5 +++++ internal/integrations/integrations.go | 1 + internal/integrations/localfs.go | 19 +++++++++++++++++++ internal/integrations/webdav.go | 19 +++++++++++++++++++ internal/messages/messages.go | 11 +++++++---- 6 files changed, 70 insertions(+), 7 deletions(-) diff --git a/internal/app/handlers.go b/internal/app/handlers.go index 4773eb96..e7e76a4b 100644 --- a/internal/app/handlers.go +++ b/internal/app/handlers.go @@ -908,9 +908,25 @@ func (app *App) blobStorageWrite(c *gin.Context) { } func (app *App) integrationsGetMetadata(c *gin.Context) { - var metadata messages.IntegrationMetadata - metadata.Thumbnail = "" - c.JSON(http.StatusOK, &metadata) + uid := c.GetString(userIDKey) + integrationID := common.ParamS(integrationKey, c) + fileID := common.ParamS(fileKey, c) + + integrationProvider, err := integrations.GetIntegrationProvider(app.userStorer, uid, integrationID) + if err != nil { + log.Error(fmt.Errorf("can't get integration, %v", err)) + c.AbortWithStatus(http.StatusInternalServerError) + return + } + + metadata, err := integrationProvider.GetMetadata(fileID) + if err != nil { + log.Error(err) + c.AbortWithStatus(http.StatusInternalServerError) + return + } + + c.JSON(http.StatusOK, metadata) } func (app *App) integrationsUpload(c *gin.Context) { diff --git a/internal/integrations/dropbox.go b/internal/integrations/dropbox.go index 41f7113f..7e06f0d8 100644 --- a/internal/integrations/dropbox.go +++ b/internal/integrations/dropbox.go @@ -24,6 +24,11 @@ func newDropbox(i model.IntegrationConfig) *DropBox { } } +func (d *DropBox) GetMetadata(fileID string) (*messages.IntegrationMetadata, error) { + // TODO + return nil, nil +} + func (d *DropBox) List(folderID string, depth int) (*messages.IntegrationFolder, error) { args := files.ListFolderArg{ diff --git a/internal/integrations/integrations.go b/internal/integrations/integrations.go index b4f59dc5..79ddfbe0 100644 --- a/internal/integrations/integrations.go +++ b/internal/integrations/integrations.go @@ -21,6 +21,7 @@ const ( // IntegrationProvider abstracts 3rd party integrations type IntegrationProvider interface { + GetMetadata(fileID string) (result *messages.IntegrationMetadata, err error) List(folderID string, depth int) (result *messages.IntegrationFolder, err error) Download(fileID string) (io.ReadCloser, error) Upload(folderID, name, fileType string, reader io.ReadCloser) (string, error) diff --git a/internal/integrations/localfs.go b/internal/integrations/localfs.go index b194619d..884fab73 100644 --- a/internal/integrations/localfs.go +++ b/internal/integrations/localfs.go @@ -26,6 +26,25 @@ func newLocalFS(i model.IntegrationConfig) *localFS { } } +func (d *localFS) GetMetadata(fileID string) (*messages.IntegrationMetadata, error) { + decoded, err := decodeName(fileID) + if err != nil { + return nil, err + } + + ext := path.Ext(decoded) + contentType := contentTypeFromExt(ext) + + return &messages.IntegrationMetadata{ + ID: fileID, + Name: path.Base(decoded), + Thumbnail: []byte{}, + SourceFileType: contentType, + ProvidedFileType: contentType, + FileType: ext, + }, nil +} + // List populates the response func (d *localFS) List(folder string, depth int) (*messages.IntegrationFolder, error) { response := messages.NewIntegrationFolder(folder, "") diff --git a/internal/integrations/webdav.go b/internal/integrations/webdav.go index 74ab2900..6584f979 100644 --- a/internal/integrations/webdav.go +++ b/internal/integrations/webdav.go @@ -97,6 +97,25 @@ func (w *WebDavIntegration) Download(fileID string) (io.ReadCloser, error) { return w.c.ReadStream(decoded) } +func (w *WebDavIntegration) GetMetadata(fileID string) (*messages.IntegrationMetadata, error) { + decoded, err := decodeName(fileID) + if err != nil { + return nil, err + } + + ext := path.Ext(decoded) + contentType := contentTypeFromExt(ext) + + return &messages.IntegrationMetadata{ + ID: fileID, + Name: path.Base(decoded), + Thumbnail: []byte{}, + SourceFileType: contentType, + ProvidedFileType: contentType, + FileType: ext, + }, nil +} + // List populates the response func (w *WebDavIntegration) List(folder string, depth int) (*messages.IntegrationFolder, error) { response := messages.NewIntegrationFolder(folder, "") diff --git a/internal/messages/messages.go b/internal/messages/messages.go index 87651ff6..77985ed0 100644 --- a/internal/messages/messages.go +++ b/internal/messages/messages.go @@ -225,8 +225,11 @@ type IntegrationFolder struct { } type IntegrationMetadata struct { - FileType string `json:"fileType"` - ID string `json:"id"` - Name string `json:"name"` - Thumbnail string `json:"thumbnail"` + ID string `json:"id"` + Name string `json:"name"` + // Thumbnail is base64 encoded string of an image/png + Thumbnail []byte `json:"thumbnail"` + SourceFileType string `json:"sourceFileType"` + ProvidedFileType string `json:"providedFileType"` + FileType string `json:"fileType"` } From adefc34943400e5dcf9152d73bcebb7678ec373c Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sun, 17 Nov 2024 15:05:05 +0100 Subject: [PATCH 4/6] integrations: Add Size in return to download interface --- internal/app/handlers.go | 5 +++-- internal/integrations/dropbox.go | 4 ++-- internal/integrations/integrations.go | 3 +-- internal/integrations/localfs.go | 13 ++++++++++--- internal/integrations/webdav.go | 14 +++++++++++--- 5 files changed, 27 insertions(+), 12 deletions(-) diff --git a/internal/app/handlers.go b/internal/app/handlers.go index e7e76a4b..0557c831 100644 --- a/internal/app/handlers.go +++ b/internal/app/handlers.go @@ -968,7 +968,7 @@ func (app *App) integrationsGetFile(c *gin.Context) { return } - reader, err := integrationProvider.Download(fileID) + reader, size, err := integrationProvider.Download(fileID) if err != nil { log.Error(err) c.AbortWithStatus(http.StatusInternalServerError) @@ -977,8 +977,9 @@ func (app *App) integrationsGetFile(c *gin.Context) { defer reader.Close() - c.DataFromReader(http.StatusOK, -1, "application/octet-stream", reader, nil) + c.DataFromReader(http.StatusOK, size, "application/octet-stream", reader, nil) } + func (app *App) integrationsList(c *gin.Context) { uid := c.GetString(userIDKey) integrationID := common.ParamS(integrationKey, c) diff --git a/internal/integrations/dropbox.go b/internal/integrations/dropbox.go index 7e06f0d8..0e539eb6 100644 --- a/internal/integrations/dropbox.go +++ b/internal/integrations/dropbox.go @@ -70,8 +70,8 @@ func (d *DropBox) List(folderID string, depth int) (*messages.IntegrationFolder, return response, nil } -func (d *DropBox) Download(fileID string) (io.ReadCloser, error) { - return nil, nil +func (d *DropBox) Download(fileID string) (io.ReadCloser, int64, error) { + return nil, 0, nil } func (d *DropBox) Upload(folderID, name, fileType string, reader io.ReadCloser) (string, error) { diff --git a/internal/integrations/integrations.go b/internal/integrations/integrations.go index 79ddfbe0..0dfe29d0 100644 --- a/internal/integrations/integrations.go +++ b/internal/integrations/integrations.go @@ -23,7 +23,7 @@ const ( type IntegrationProvider interface { GetMetadata(fileID string) (result *messages.IntegrationMetadata, err error) List(folderID string, depth int) (result *messages.IntegrationFolder, err error) - Download(fileID string) (io.ReadCloser, error) + Download(fileID string) (io.ReadCloser, int64, error) Upload(folderID, name, fileType string, reader io.ReadCloser) (string, error) } @@ -126,7 +126,6 @@ func visitDir(root, currentPath string, depth int, parentFolder *messages.Integr docName := strings.TrimSuffix(entryName, ext) extension := strings.TrimPrefix(ext, ".") - file := &messages.IntegrationFile{ ProvidedFileType: contentType, DateChanged: d.ModTime(), diff --git a/internal/integrations/localfs.go b/internal/integrations/localfs.go index 884fab73..2470903c 100644 --- a/internal/integrations/localfs.go +++ b/internal/integrations/localfs.go @@ -88,16 +88,23 @@ func (d *localFS) List(folder string, depth int) (*messages.IntegrationFolder, e return response, nil } -func (d *localFS) Download(fileID string) (io.ReadCloser, error) { +func (d *localFS) Download(fileID string) (io.ReadCloser, int64, error) { decoded, err := decodeName(fileID) if err != nil { - return nil, err + return nil, 0, err } localPath := path.Join(d.rootPath, path.Clean(decoded)) - return os.Open(localPath) + st, err := os.Stat(localPath) + if err != nil { + return nil, 0, err + } + + res, err := os.Open(localPath) + return res, st.Size(), err } + func (d *localFS) Upload(folderID, name, fileType string, reader io.ReadCloser) (id string, err error) { folder := "/" if folderID != rootFolder { diff --git a/internal/integrations/webdav.go b/internal/integrations/webdav.go index 6584f979..267966d1 100644 --- a/internal/integrations/webdav.go +++ b/internal/integrations/webdav.go @@ -89,12 +89,20 @@ func (w *WebDavIntegration) Upload(folderID, name, fileType string, reader io.Re } // Download downloads -func (w *WebDavIntegration) Download(fileID string) (io.ReadCloser, error) { +func (w *WebDavIntegration) Download(fileID string) (io.ReadCloser, int64, error) { decoded, err := decodeName(fileID) if err != nil { - return nil, err + return nil, 0, err + } + + st, err := w.c.Stat(decoded) + if err != nil { + return nil, 0, err } - return w.c.ReadStream(decoded) + + res, err := w.c.ReadStream(decoded) + + return res, st.Size(), err } func (w *WebDavIntegration) GetMetadata(fileID string) (*messages.IntegrationMetadata, error) { From a5bd8c2ae78a49f039d52509134c4c471b80041f Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sun, 17 Nov 2024 15:05:59 +0100 Subject: [PATCH 5/6] Add API routes to explore integrations content in ui --- internal/ui/handlers.go | 78 +++++++++++++++++++++++++++++++++++++++++ internal/ui/routes.go | 4 +++ 2 files changed, 82 insertions(+) diff --git a/internal/ui/handlers.go b/internal/ui/handlers.go index e30e7823..09e7686d 100644 --- a/internal/ui/handlers.go +++ b/internal/ui/handlers.go @@ -6,6 +6,7 @@ import ( "time" "github.com/ddvk/rmfakecloud/internal/common" + "github.com/ddvk/rmfakecloud/internal/integrations" "github.com/ddvk/rmfakecloud/internal/model" "github.com/ddvk/rmfakecloud/internal/storage" "github.com/ddvk/rmfakecloud/internal/ui/viewmodel" @@ -648,3 +649,80 @@ func (app *ReactAppWrapper) deleteIntegration(c *gin.Context) { c.AbortWithStatus(http.StatusNotFound) } + +func (app *ReactAppWrapper) exploreIntegration(c *gin.Context) { + uid := c.GetString(userIDContextKey) + + integrationID := common.ParamS(intIDParam, c) + + integrationProvider, err := integrations.GetIntegrationProvider(app.userStorer, uid, integrationID) + if err != nil { + log.Error(err) + c.AbortWithStatus(http.StatusInternalServerError) + return + } + + folder := common.ParamS("path", c) + if folder == "" { + folder = "root" + } + + response, err := integrationProvider.List(folder, 2) + if err != nil { + log.Error(err) + c.AbortWithStatus(http.StatusInternalServerError) + return + } + + c.JSON(http.StatusOK, response) +} + +func (app *ReactAppWrapper) getMetadataIntegration(c *gin.Context) { + uid := c.GetString(userIDContextKey) + + integrationID := common.ParamS(intIDParam, c) + + integrationProvider, err := integrations.GetIntegrationProvider(app.userStorer, uid, integrationID) + if err != nil { + log.Error(err) + c.AbortWithStatus(http.StatusInternalServerError) + return + } + + fileid := common.ParamS("path", c) + + response, err := integrationProvider.GetMetadata(fileid) + if err != nil { + log.Error(err) + c.AbortWithStatus(http.StatusInternalServerError) + return + } + + c.JSON(http.StatusOK, response) +} + +func (app *ReactAppWrapper) downloadThroughIntegration(c *gin.Context) { + uid := c.GetString(userIDContextKey) + + integrationID := common.ParamS(intIDParam, c) + + integrationProvider, err := integrations.GetIntegrationProvider(app.userStorer, uid, integrationID) + if err != nil { + log.Error(err) + c.AbortWithStatus(http.StatusInternalServerError) + return + } + + fileid := common.ParamS("path", c) + + response, size, err := integrationProvider.Download(fileid) + if err != nil { + log.Error(err) + c.AbortWithStatus(http.StatusInternalServerError) + return + } + + defer response.Close() + + c.DataFromReader(http.StatusOK, size, "", response, nil) +} diff --git a/internal/ui/routes.go b/internal/ui/routes.go index 4df7fd14..5a6f16fc 100644 --- a/internal/ui/routes.go +++ b/internal/ui/routes.go @@ -79,6 +79,10 @@ func (app *ReactAppWrapper) RegisterRoutes(router *gin.Engine) { auth.PUT("integrations/:intid", app.updateIntegration) auth.DELETE("integrations/:intid", app.deleteIntegration) + auth.GET("integrations/:intid/explore/*path", app.exploreIntegration) + auth.GET("integrations/:intid/metadata/*path", app.getMetadataIntegration) + auth.GET("integrations/:intid/download/*path", app.downloadThroughIntegration) + //admin admin := auth.Group("") admin.Use(app.adminMiddleware()) From 86ab6e1577c63d81fac38502a61b4ac4ad3c9e0d Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 2 Jun 2023 13:21:17 +0200 Subject: [PATCH 6/6] Add FTP integration --- go.mod | 1 + go.sum | 2 + internal/integrations/ftp.go | 134 ++++++++++++++++++ internal/integrations/integrations.go | 5 + internal/model/user.go | 5 +- ui/src/pages/Integrations/IntegrationModal.js | 18 ++- .../pages/Integrations/NewIntegrationModal.js | 16 ++- 7 files changed, 174 insertions(+), 7 deletions(-) create mode 100644 internal/integrations/ftp.go diff --git a/go.mod b/go.mod index dca34d89..40febd4f 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/juruen/rmapi v0.0.25 github.com/poundifdef/go-remarkable2pdf v0.2.0 github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 + github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.9.0 github.com/studio-b12/gowebdav v0.9.0 diff --git a/go.sum b/go.sum index 4e2d5998..7c88f196 100644 --- a/go.sum +++ b/go.sum @@ -198,6 +198,8 @@ github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 h1:mZHayPoR0lNmnH github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5/go.mod h1:GEXHk5HgEKCvEIIrSpFI3ozzG5xOKA2DVlEX/gGnewM= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= +github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo= +github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY= github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= diff --git a/internal/integrations/ftp.go b/internal/integrations/ftp.go new file mode 100644 index 00000000..c5234f26 --- /dev/null +++ b/internal/integrations/ftp.go @@ -0,0 +1,134 @@ +package integrations + +import ( + "bytes" + "io" + "os" + "path" + "strings" + + "github.com/ddvk/rmfakecloud/internal/messages" + "github.com/ddvk/rmfakecloud/internal/model" + "github.com/secsy/goftp" + "github.com/sirupsen/logrus" +) + +type FTPIntegration struct { + client *goftp.Client +} + +func newFTP(i model.IntegrationConfig) *FTPIntegration { + config := goftp.Config{ + Logger: os.Stderr, + ActiveTransfers: i.ActiveTransfers, + } + + if i.Username != "" { + config.User = i.Username + } + if i.Password != "" { + config.Password = i.Password + } + + if strings.HasPrefix(i.Address, "ftps://") { + config.TLSMode = goftp.TLSImplicit + i.Address = strings.TrimPrefix(i.Address, "ftps://") + } else if strings.HasPrefix(i.Address, "ftpes://") { + config.TLSMode = goftp.TLSExplicit + i.Address = strings.TrimPrefix(i.Address, "ftpes://") + } + + client, err := goftp.DialConfig(config, strings.TrimPrefix(i.Address, "ftp://")) + if err != nil { + logrus.Errorf("An error occurred creating FTP client: %v\n", err) + return nil + } + + return &FTPIntegration{ + client, + } +} + +func (g *FTPIntegration) GetMetadata(fileID string) (*messages.IntegrationMetadata, error) { + decoded, err := decodeName(fileID) + if err != nil { + return nil, err + } + + ext := path.Ext(decoded) + contentType := contentTypeFromExt(ext) + + return &messages.IntegrationMetadata{ + ID: fileID, + Name: path.Base(decoded), + Thumbnail: []byte{}, + SourceFileType: contentType, + ProvidedFileType: contentType, + FileType: ext, + }, nil +} + +func (g *FTPIntegration) List(folder string, depth int) (*messages.IntegrationFolder, error) { + response := messages.NewIntegrationFolder(folder, "") + + if folder == rootFolder { + folder = "/" + response.Name = "FTP root" + } else { + decoded, err := decodeName(folder) + if err != nil { + return nil, err + } + folder = decoded + response.Name = path.Base(folder) + } + logrus.Info("[ftp] query for: ", folder, " depth: ", depth) + + err := visitDir("", folder, depth, response, g.client.ReadDir) + if err != nil { + return nil, err + } + + return response, nil +} + +func (g *FTPIntegration) Download(fileID string) (io.ReadCloser, int64, error) { + decoded, err := decodeName(fileID) + if err != nil { + return nil, 0, err + } + + st, err := g.client.Stat(decoded) + if err != nil { + return nil, 0, err + } + + var buf bytes.Buffer + + err = g.client.Retrieve(decoded, &buf) + if err != nil { + return nil, st.Size(), err + } + + return io.NopCloser(&buf), st.Size(), err +} + +func (g *FTPIntegration) Upload(folderID, name, fileType string, reader io.ReadCloser) (id string, err error) { + folder := "/" + if folderID != rootFolder { + folder, err = decodeName(folderID) + if err != nil { + return + } + } + fullpath := path.Join(folder, name+"."+fileType) + logrus.Trace(logger, "Uploading: ", fullpath) + + err = g.client.Store(fullpath, reader) + + if err != nil { + return + } + id = encodeName(fullpath) + return +} diff --git a/internal/integrations/integrations.go b/internal/integrations/integrations.go index 0dfe29d0..35f05d28 100644 --- a/internal/integrations/integrations.go +++ b/internal/integrations/integrations.go @@ -13,6 +13,7 @@ import ( ) const ( + ftpProvider = "ftp" webdavProvider = "webdav" dropboxProvider = "dropbox" googleProvider = "google" @@ -40,6 +41,8 @@ func GetIntegrationProvider(storer storage.UserStorer, uid, integrationid string switch intg.Provider { case dropboxProvider: return newDropbox(intg), nil + case ftpProvider: + return newFTP(intg), nil case localfsProvider: return newLocalFS(intg), nil case webdavProvider: @@ -53,6 +56,8 @@ func GetIntegrationProvider(storer storage.UserStorer, uid, integrationid string // fix the name func fixProviderName(n string) string { switch n { + case ftpProvider: + fallthrough case dropboxProvider: return "Dropbox" case googleProvider: diff --git a/internal/model/user.go b/internal/model/user.go index d6fba366..b736ab0f 100644 --- a/internal/model/user.go +++ b/internal/model/user.go @@ -63,11 +63,14 @@ type IntegrationConfig struct { Provider string Name string - // WebDav + // WebDav // FTP Username string Password string Address string + // FTP + ActiveTransfers bool + // Insecure ignore TLS cert errors Insecure bool diff --git a/ui/src/pages/Integrations/IntegrationModal.js b/ui/src/pages/Integrations/IntegrationModal.js index a2f9d156..e4c89843 100644 --- a/ui/src/pages/Integrations/IntegrationModal.js +++ b/ui/src/pages/Integrations/IntegrationModal.js @@ -16,6 +16,7 @@ export default function IntegrationModal(params) { username: integration?.Username, password: integration?.Password, address: integration?.Address, + activetransfers: integration?.ActiveTransfers, insecure: integration?.Insecure, accesstoken: integration?.Accesstoken, path: integration?.Path, @@ -48,6 +49,7 @@ export default function IntegrationModal(params) { username: integrationForm.username, password: integrationForm.password, address: integrationForm.address, + activetransfers: integrationForm?.activetransfers, insecure: integrationForm.insecure, accesstoken: integrationForm.accesstoken, path: integrationForm.path, @@ -89,6 +91,7 @@ export default function IntegrationModal(params) { > + @@ -100,7 +103,7 @@ export default function IntegrationModal(params) { onChange={handleChange} /> - {integrationForm.provider === "webdav" && ( + {(integrationForm.provider === "webdav" || integrationForm.provider === "ftp") && ( <> Address )} - {integrationForm.provider === "webdav" && ( + {(integrationForm.provider === "webdav" || integrationForm.provider === "ftp") && ( <> Username )} - {integrationForm.provider === "webdav" && ( + {(integrationForm.provider === "webdav" || integrationForm.provider === "ftp") && ( <> Password )} + {integrationForm.provider === "ftp" && ( + setIntegrationForm({ ...integrationForm, [target.name]: target.checked })} + label="Use actives transfers" + /> + )} + {integrationForm.provider === "localfs" && ( <> Path diff --git a/ui/src/pages/Integrations/NewIntegrationModal.js b/ui/src/pages/Integrations/NewIntegrationModal.js index 7490f37b..68965a36 100644 --- a/ui/src/pages/Integrations/NewIntegrationModal.js +++ b/ui/src/pages/Integrations/NewIntegrationModal.js @@ -78,11 +78,12 @@ export default function IntegrationProfileModal(params) { className="mb-1" > + - {integrationForm.provider === "webdav" && ( + {(integrationForm.provider === "webdav" || integrationForm.provider === "ftp") && ( <> Address )} - {integrationForm.provider === "webdav" && ( + {(integrationForm.provider === "webdav" || integrationForm.provider === "ftp") && ( <> Username )} - {integrationForm.provider === "webdav" && ( + {(integrationForm.provider === "webdav" || integrationForm.provider === "ftp") && ( <> Password )} + {integrationForm.provider === "ftp" && ( + setIntegrationForm({ ...integrationForm, [target.name]: target.checked })} + label="Use actives transfers" + /> + )} + {integrationForm.provider === "localfs" && ( <> Path