From eab3df77a608b9d4128911cfb61fb4a3b3bf0aa6 Mon Sep 17 00:00:00 2001 From: hiroshisuga <45039819+hiroshisuga@users.noreply.github.com> Date: Sun, 28 May 2023 20:44:53 +0900 Subject: [PATCH 01/88] Update pushSpeechTranscript.js --- .../server/methods/pushSpeechTranscript.js | 83 +++++++++++++++---- 1 file changed, 69 insertions(+), 14 deletions(-) diff --git a/bigbluebutton-html5/imports/api/captions/server/methods/pushSpeechTranscript.js b/bigbluebutton-html5/imports/api/captions/server/methods/pushSpeechTranscript.js index 7e81127fe6f2..833880429f17 100644 --- a/bigbluebutton-html5/imports/api/captions/server/methods/pushSpeechTranscript.js +++ b/bigbluebutton-html5/imports/api/captions/server/methods/pushSpeechTranscript.js @@ -1,11 +1,67 @@ import { check } from 'meteor/check'; import Captions from '/imports/api/captions'; +import Users from '/imports/api/users'; import { extractCredentials } from '/imports/api/common/server/helpers'; import Logger from '/imports/startup/server/logger'; import setTranscript from '/imports/api/captions/server/modifiers/setTranscript'; import updatePad from '/imports/api/pads/server/methods/updatePad'; +import axios from 'axios'; -export default function pushSpeechTranscript(locale, transcript, type) { +const CAPTIONS_CONFIG = Meteor.settings.public.captions; + +function sendTranscript(meetingId, requesterUserId, type, dst, text) { + const user = Users.findOne({meetingId, userId: requesterUserId}, { fields: {name: 1}}); + //console.log("sendTranscript", text); + const textWithName = `${user.name}: ${text}`; + if (type === 'final') { + const textLf = `\n${textWithName}`; + updatePad(meetingId, requesterUserId, dst, textLf); + } + setTranscript(meetingId, dst, textWithName); +} + +function translateText(meetingId, requesterUserId, textOri, type, src, dst) { + if ( !CAPTIONS_CONFIG.enableAutomaticTranslation || textOri === "" || !dst || dst === "" || dst === src || dst.replace(/-..$/,'') === src || dst === src.replace(/-..$/,'') ) { + sendTranscript(meetingId, requesterUserId, type, dst, textOri); + } else { + let url = ''; + if (CAPTIONS_CONFIG.googleTranslateUrl) { + url = CAPTIONS_CONFIG.googleTranslateUrl + '/exec?' + + 'text=' + encodeURIComponent(textOri) + '&source=' + src + '&target=' + dst; + } else if (CAPTIONS_CONFIG.deeplTranslateUrl) { + url = CAPTIONS_CONFIG.deeplTranslateUrl + + '&text=' + encodeURIComponent(textOri) + '&source_lang=' + src.replace(/-..$/,'').toUpperCase() + + '&target_lang=' + dst.toUpperCase(); + } else { + Logger.error('Could not get a translation service.'); + return; + } + + axios({ + method: 'get', + url, + responseType: 'json', + }).then((response) => { + if (CAPTIONS_CONFIG.googleTranslateUrl) { + const { code, text } = response.data; + if (code === 200) { + sendTranscript(meetingId, requesterUserId, type, dst, text); + } else { + Logger.error(`Failed to get Google translation for ${textOri}`); + } + } else if (CAPTIONS_CONFIG.deeplTranslateUrl) { + const { translations } = response.data; + if (translations.length > 0 && translations[0].text) { + sendTranscript(meetingId, requesterUserId, type, dst, translations[0].text); + } else { + Logger.error(`Failed to get DeepL translation for ${textOri}`); + } + } + }).catch((error) => Logger.error(`Could not get translation for ${textOri.trim()} on the locale ${dst}: ${error}`)); + } +} + +export default function pushSpeechTranscript(locale, transcript, type, locales) { try { const { meetingId, requesterUserId } = extractCredentials(this.userId); @@ -14,22 +70,21 @@ export default function pushSpeechTranscript(locale, transcript, type) { check(locale, String); check(transcript, String); check(type, String); + check(locales, Array); - const captions = Captions.findOne({ - meetingId, - ownerId: requesterUserId, - locale, - dictating: true, - }); + locales.forEach(function(dstLocale, index) { + const caption = Captions.findOne({ + meetingId, + locale: dstLocale, + }); - if (captions) { - if (type === 'final') { - const text = `\n${transcript}`; - updatePad(meetingId, requesterUserId, locale, text); + if (!caption) { + Logger.error(`Could not find caption's pad for meetingId=${meetingId} locale=${locale}`); + return; + } else { + translateText(meetingId, requesterUserId, transcript, type, locale, dstLocale); } - - setTranscript(meetingId, locale, transcript); - } + }); } catch (err) { Logger.error(`Exception while invoking method pushSpeechTranscript ${err.stack}`); } From ca55fc2dd8b63a0043ee6fd76ed8bdd0e539f808 Mon Sep 17 00:00:00 2001 From: hiroshisuga <45039819+hiroshisuga@users.noreply.github.com> Date: Sun, 28 May 2023 20:45:39 +0900 Subject: [PATCH 02/88] Update startDictation.js --- .../imports/api/captions/server/methods/startDictation.js | 1 - 1 file changed, 1 deletion(-) diff --git a/bigbluebutton-html5/imports/api/captions/server/methods/startDictation.js b/bigbluebutton-html5/imports/api/captions/server/methods/startDictation.js index 7d0baa2a9df9..eef995558144 100644 --- a/bigbluebutton-html5/imports/api/captions/server/methods/startDictation.js +++ b/bigbluebutton-html5/imports/api/captions/server/methods/startDictation.js @@ -14,7 +14,6 @@ export default function startDictation(locale) { const captions = Captions.findOne({ meetingId, - ownerId: requesterUserId, locale, }); From c29c3046d5522958d1e7c1804591d2376f32ffe5 Mon Sep 17 00:00:00 2001 From: hiroshisuga <45039819+hiroshisuga@users.noreply.github.com> Date: Sun, 28 May 2023 20:45:52 +0900 Subject: [PATCH 03/88] Update stopDictation.js --- .../imports/api/captions/server/methods/stopDictation.js | 1 - 1 file changed, 1 deletion(-) diff --git a/bigbluebutton-html5/imports/api/captions/server/methods/stopDictation.js b/bigbluebutton-html5/imports/api/captions/server/methods/stopDictation.js index 44e469348ad1..7f8078dfdbcf 100644 --- a/bigbluebutton-html5/imports/api/captions/server/methods/stopDictation.js +++ b/bigbluebutton-html5/imports/api/captions/server/methods/stopDictation.js @@ -14,7 +14,6 @@ export default function stopDictation(locale) { const captions = Captions.findOne({ meetingId, - ownerId: requesterUserId, locale, }); From 9965c43c4d48684dc348c83db4d708280580839a Mon Sep 17 00:00:00 2001 From: hiroshisuga <45039819+hiroshisuga@users.noreply.github.com> Date: Sun, 28 May 2023 20:47:16 +0900 Subject: [PATCH 04/88] Update createCaptions.js --- .../imports/api/captions/server/modifiers/createCaptions.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bigbluebutton-html5/imports/api/captions/server/modifiers/createCaptions.js b/bigbluebutton-html5/imports/api/captions/server/modifiers/createCaptions.js index 382be4726615..f92b3ea8fd58 100644 --- a/bigbluebutton-html5/imports/api/captions/server/modifiers/createCaptions.js +++ b/bigbluebutton-html5/imports/api/captions/server/modifiers/createCaptions.js @@ -20,6 +20,9 @@ export default function createCaptions(meetingId, locale, name) { ownerId: '', dictating: false, transcript: '', + translating: false, + translationDoner: {}, + speechDoner: {}, }; const numberAffected = Captions.upsert(selector, modifier); From 03d06f7a3cdcebca837511600616bb63a64fa7c9 Mon Sep 17 00:00:00 2001 From: hiroshisuga <45039819+hiroshisuga@users.noreply.github.com> Date: Sun, 28 May 2023 20:47:55 +0900 Subject: [PATCH 05/88] Update updateCaptionsOwner.js --- .../imports/api/captions/server/modifiers/updateCaptionsOwner.js | 1 - 1 file changed, 1 deletion(-) diff --git a/bigbluebutton-html5/imports/api/captions/server/modifiers/updateCaptionsOwner.js b/bigbluebutton-html5/imports/api/captions/server/modifiers/updateCaptionsOwner.js index 61595c886f9f..e4a10524920f 100644 --- a/bigbluebutton-html5/imports/api/captions/server/modifiers/updateCaptionsOwner.js +++ b/bigbluebutton-html5/imports/api/captions/server/modifiers/updateCaptionsOwner.js @@ -16,7 +16,6 @@ export default function updateCaptionsOwner(meetingId, locale, ownerId) { const modifier = { $set: { ownerId, - dictating: false, // Refresh dictation mode }, }; From cc82223420bc3a6fd1a137394007b1aa872581aa Mon Sep 17 00:00:00 2001 From: hiroshisuga <45039819+hiroshisuga@users.noreply.github.com> Date: Sun, 28 May 2023 20:52:46 +0900 Subject: [PATCH 06/88] Update component.jsx --- .../components/captions/button/component.jsx | 74 ++++++++++++++++--- 1 file changed, 62 insertions(+), 12 deletions(-) diff --git a/bigbluebutton-html5/imports/ui/components/captions/button/component.jsx b/bigbluebutton-html5/imports/ui/components/captions/button/component.jsx index 19de6750f15a..f3780df19b0b 100644 --- a/bigbluebutton-html5/imports/ui/components/captions/button/component.jsx +++ b/bigbluebutton-html5/imports/ui/components/captions/button/component.jsx @@ -2,6 +2,15 @@ import React from 'react'; import PropTypes from 'prop-types'; import { defineMessages, injectIntl } from 'react-intl'; import Styled from './styles'; +import ButtonEmoji from '/imports/ui/components/common/button/button-emoji/ButtonEmoji'; +import _ from 'lodash'; +import Dropdown from '/imports/ui/components/dropdown/component'; +import DropdownTrigger from '/imports/ui/components/dropdown/trigger/component'; +import DropdownContent from '/imports/ui/components/dropdown/content/component'; +import DropdownList from '/imports/ui/components/dropdown/list/component'; +import DropdownListItem from '/imports/ui/components/dropdown/list/item/component'; +import Storage from '/imports/ui/services/storage/session'; +import Service from '/imports/ui/components/captions/service'; const propTypes = { intl: PropTypes.shape({ @@ -20,20 +29,61 @@ const intlMessages = defineMessages({ id: 'app.actionsBar.captions.stop', description: 'Stop closed captions option', }, + changeCaption: { + id: 'app.actionsBar.changeCaption', + description: 'Open change caption label', + }, }); -const CaptionsButton = ({ intl, isActive, handleOnClick }) => ( - +const handleClickCaption = (locale, selectedLocale) => { + if (selectedLocale != locale) { + Service.setCaptionsActive(locale); + } +} + +const getAvailableCaptions = (captions, selected) => { + return( + captions.map(locale => ( + handleClickCaption(locale.locale, selected)} + iconRight={selected == locale.locale ? 'check' : null} + /> + )) + ); +}; + +const CaptionsButton = ({ intl, isActive, handleOnClick, translatedLocales, selectedLocale }) => ( +
+ + {isActive && + + + + + + + {getAvailableCaptions(translatedLocales, selectedLocale)} + + + + } +
); CaptionsButton.propTypes = propTypes; From 9f4e84f7abdeacc88d63445d2737d9b6d704fe9f Mon Sep 17 00:00:00 2001 From: hiroshisuga <45039819+hiroshisuga@users.noreply.github.com> Date: Sun, 28 May 2023 20:54:26 +0900 Subject: [PATCH 07/88] Update container.jsx --- .../imports/ui/components/captions/button/container.jsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bigbluebutton-html5/imports/ui/components/captions/button/container.jsx b/bigbluebutton-html5/imports/ui/components/captions/button/container.jsx index add5eb0a6673..064a8b410f03 100644 --- a/bigbluebutton-html5/imports/ui/components/captions/button/container.jsx +++ b/bigbluebutton-html5/imports/ui/components/captions/button/container.jsx @@ -12,4 +12,6 @@ export default withModalMounter(withTracker(({ mountModal }) => ({ handleOnClick: () => (Service.isCaptionsActive() ? Service.deactivateCaptions() : mountModal()), + translatedLocales: Service.getLocalesAutoTranslated(), + selectedLocale: Service.getCaptionsActive(), }))(Container)); From d6fd312ef1227a68fe6b514d755da3f336d62f38 Mon Sep 17 00:00:00 2001 From: hiroshisuga <45039819+hiroshisuga@users.noreply.github.com> Date: Sun, 28 May 2023 21:04:24 +0900 Subject: [PATCH 08/88] Update component.jsx --- .../ui/components/captions/component.jsx | 111 ++++++++++++++---- 1 file changed, 90 insertions(+), 21 deletions(-) diff --git a/bigbluebutton-html5/imports/ui/components/captions/component.jsx b/bigbluebutton-html5/imports/ui/components/captions/component.jsx index 3b5f4a5a82b5..99c989ade055 100644 --- a/bigbluebutton-html5/imports/ui/components/captions/component.jsx +++ b/bigbluebutton-html5/imports/ui/components/captions/component.jsx @@ -9,6 +9,9 @@ import Styled from './styles'; import { PANELS, ACTIONS } from '/imports/ui/components/layout/enums'; import browserInfo from '/imports/utils/browserInfo'; import Header from '/imports/ui/components/common/control-header/component'; +import Toggle from '/imports/ui/components/common/switch/component'; +import StyledHeader from '/imports/ui/components/common/control-header//styles'; +import { components } from 'react-select'; const intlMessages = defineMessages({ hide: { @@ -39,17 +42,22 @@ const intlMessages = defineMessages({ id: 'app.captions.dictationOffDesc', description: 'Aria description for button that turns off speech recognition', }, + autoTranslation: { + id: 'app.captions.pad.autoTranslation', + description: 'Label for auto translation of closed captions pad', + }, + autoTranslationDesc: { + id: 'app.captions.pad.autoTranslationDesc', + description: 'Descriotion for auto translation of closed captions pad', + }, }); const propTypes = { locale: PropTypes.string.isRequired, name: PropTypes.string.isRequired, - ownerId: PropTypes.string.isRequired, intl: PropTypes.shape({ formatMessage: PropTypes.func.isRequired, }).isRequired, - dictation: PropTypes.bool.isRequired, - dictating: PropTypes.bool.isRequired, isRTL: PropTypes.bool.isRequired, hasPermission: PropTypes.bool.isRequired, layoutContextDispatch: PropTypes.func.isRequired, @@ -59,17 +67,57 @@ const propTypes = { const Captions = ({ locale, intl, - ownerId, name, - dictation, - dictating, + amISpeaker, isRTL, hasPermission, layoutContextDispatch, isResizing, + //isAutoTranslated, + //toggleAutoTranslation, }) => { const { isChrome } = browserInfo; + const localeOptions = []; + const selectedLocales = []; + Service.getAvailableLocales().forEach((loc) => { + //The current locale not included + localeOptions.push({value: loc.locale, label: loc.name}); + if (loc.translating) { + if (loc.locale == locale) { + selectedLocales.push({value: loc.locale, label: loc.name, isFixed: true}); + } else { + selectedLocales.push({value: loc.locale, label: loc.name}); + } + } + }); + + const onTranslationLocaleChanged = ( + newValue, + actionMeta + ) => { + switch (actionMeta.action) { + case 'remove-value': + case 'pop-value': + Service.removeTranslation(actionMeta.removedValue.value); + break; + case 'select-option': + Service.selectTranslation(actionMeta.option.value); + break; + case 'clear': + //This would't happen.. + Service.clearTranslation(); + break; + } + }; + + const MultiValueRemove = (props) => { + if (props.data.isFixed) { + return null; + } + return ; + }; + return (