From 30ecab012747af84b23fbbb4e42f1b6876c48c2f Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Tue, 12 Dec 2023 22:26:13 -0800 Subject: [PATCH 001/328] Adding logic to better handle different types or redirects and websocket protocols. --- octoeverywhere/WebStream/octoheaderimpl.py | 54 +++++++++++++++---- .../WebStream/octowebstreamhttphelper.py | 2 +- .../WebStream/octowebstreamwshelper.py | 3 +- octoeverywhere/octostreammsgbuilder.py | 2 +- octoeverywhere/websocketimpl.py | 5 +- 5 files changed, 50 insertions(+), 16 deletions(-) diff --git a/octoeverywhere/WebStream/octoheaderimpl.py b/octoeverywhere/WebStream/octoheaderimpl.py index 2b2fab6..2f39c76 100644 --- a/octoeverywhere/WebStream/octoheaderimpl.py +++ b/octoeverywhere/WebStream/octoheaderimpl.py @@ -1,4 +1,4 @@ -import sys +import logging from ..sentry import Sentry from ..octostreammsgbuilder import OctoStreamMsgBuilder @@ -115,7 +115,7 @@ def GatherRequestHeaders(logger, httpInitialContextOptional, protocol) : # Called only for websockets to get headers. @staticmethod - def GatherWebsocketRequestHeaders(logger, httpInitialContext) : + def GatherWebsocketRequestHeaders(logger:logging.Logger, httpInitialContext) -> dict: # Get the count of headers in the message. headersLen = httpInitialContext.HeadersLength() @@ -134,22 +134,42 @@ def GatherWebsocketRequestHeaders(logger, httpInitialContext) : continue lowerName = name.lower() - # Right now we only allow the x-api-key header to be sent to the websocket. - # This is because the OctoPrint websocket server seems to fail if we send most of the standard headers. - # The only one that's known be required right now is the x-api-key, since it needed by moonraker apps if auth is enabled. + # Right now we only allow a subset of headers. Some headers seem to break the websocket servers, so we only allow the ones + # we know we need. if lowerName.startswith("x-api-key"): - # Add the header. (use the original case) + sendHeaders[name] = value + elif lowerName == "cookie": sendHeaders[name] = value return sendHeaders + # Given an httpInitialContext returns if there are any web socket subprotocols being asked for. + @staticmethod + def GetWebSocketSubProtocols(logger:logging.Logger, httpInitialContext) -> list: + # Get the count of headers in the message. + headersLen = httpInitialContext.HeadersLength() + i = 0 + while i < headersLen: + # Get the header + header = httpInitialContext.Headers(i) + i += 1 + + # Check if it's the protocol headers\ + name = OctoStreamMsgBuilder.BytesToString(header.Key()) + lowerName = name.lower() + if lowerName == "sec-websocket-protocol": + valueList = OctoStreamMsgBuilder.BytesToString(header.Value()) + return valueList.split(",") + return None + + # We have noticed that some proxy servers aren't setup correctly to forward the x-forwarded-for and such headers. # So when the web server responds back with a 301 or 302, the location header might not have the correct hostname, instead an ip like 127.0.0.1. # # This function must return the location value string again, either corrected or not. @staticmethod - def CorrectLocationResponseHeaderIfNeeded(logger, locationValue, sendHeaders): + def CorrectLocationResponseHeaderIfNeeded(logger:logging.Logger, requestUri:str, locationValue:str, sendHeaders): # The sendHeaders is an dict that was generated by GatherRequestHeaders and were used to send the request. # Make sure the location is http(s) or ws(s), since that's all we deal with right now. @@ -172,20 +192,32 @@ def CorrectLocationResponseHeaderIfNeeded(logger, locationValue, sendHeaders): urlStart = sendHeaders[HeaderHelper.c_xForwardedForProtoHeaderName] + "://" + sendHeaders[HeaderHelper.c_xForwardedForHostHeaderName] try: - # Since urlparse is only support on py3, for py2, we don't support this feature. - if sys.version_info[0] < 3: - return locationValue - # Parse the existing URL to get the path. # pylint: disable=import-outside-toplevel from urllib.parse import urlparse r = urlparse(locationValue) + # If the redirect starts with ./ it's referencing the current uri path. + # For example, if the request uri was https://test.com/hello/world and the redirect is ./overhere?test=1 + # The correct URI is https://test.com/hello/world/overhere?test=1 + path = r.path + if path.startswith("./"): + # Parse the request uri to pull the path out. + ogUri = urlparse(requestUri) + path = ogUri.path + # Ensure the path starts with a / + if path.startswith("/") is False: + path += "/" + # Append the redirect path, but not the ./ + if len(r.path) > 2: + path += r.path[2:] + # Return the new URL # The path value will start with a / if there was one in the original path. # If there was no slash (http://octoeverywhere.com) path is an empty string. # If there is no query string, it's an empty string as well. correctedUrl = urlStart + r.path + correctedUrl = urlStart + path if len(r.query) > 0: correctedUrl += "?" + r.query diff --git a/octoeverywhere/WebStream/octowebstreamhttphelper.py b/octoeverywhere/WebStream/octowebstreamhttphelper.py index f6fe326..a69b18c 100644 --- a/octoeverywhere/WebStream/octowebstreamhttphelper.py +++ b/octoeverywhere/WebStream/octowebstreamhttphelper.py @@ -275,7 +275,7 @@ def executeHttpRequest(self): elif nameLower == "location": # We have noticed that some proxy servers aren't setup correctly to forward the x-forwarded-for and such headers. # So when the web server responds back with a 301 or 302, the location header might not have the correct hostname, instead an ip like 127.0.0.1. - response.headers[name] = HeaderHelper.CorrectLocationResponseHeaderIfNeeded(self.Logger, response.headers[name], sendHeaders) + response.headers[name] = HeaderHelper.CorrectLocationResponseHeaderIfNeeded(self.Logger, uri, response.headers[name], sendHeaders) # We also look at the content-type to determine if we should add compression to this request or not. # general rule of thumb is that compression is quite cheap but really helps with text, so we should compress when we diff --git a/octoeverywhere/WebStream/octowebstreamwshelper.py b/octoeverywhere/WebStream/octowebstreamwshelper.py index 00aabac..64334c7 100644 --- a/octoeverywhere/WebStream/octowebstreamwshelper.py +++ b/octoeverywhere/WebStream/octowebstreamwshelper.py @@ -57,6 +57,7 @@ def __init__(self, streamId, logger, webStream, webStreamOpenMsg, openedTime): # Parse the headers, filter them, and keep them locally. # This is required for klipper clients, since they need to send the X-API-Key header with the API key. self.Headers = HeaderHelper.GatherWebsocketRequestHeaders(self.Logger, self.HttpInitialContext) + self.SubProtocolList = HeaderHelper.GetWebSocketSubProtocols(self.Logger, self.HttpInitialContext) # It might take multiple attempts depending on the network setup of the client. # This value keeps track of them. @@ -185,7 +186,7 @@ def AttemptConnection(self): # Make the websocket object and start it running. self.Logger.debug(self.getLogMsgPrefix()+"opening websocket to "+str(uri) + " attempt "+ str(self.ConnectionAttempt)) - self.Ws = Client(uri, self.onWsOpened, None, self.onWsData, self.onWsClosed, self.onWsError, headers=self.Headers) + self.Ws = Client(uri, self.onWsOpened, None, self.onWsData, self.onWsClosed, self.onWsError, subProtocolList=self.SubProtocolList) self.Ws.RunAsync() # Return true to indicate we are trying to connect again. diff --git a/octoeverywhere/octostreammsgbuilder.py b/octoeverywhere/octostreammsgbuilder.py index ecfdd02..44dea91 100644 --- a/octoeverywhere/octostreammsgbuilder.py +++ b/octoeverywhere/octostreammsgbuilder.py @@ -60,7 +60,7 @@ def CreateOctoStreamMsgAndFinalize(builder, contextType, contextOffset): return builder.Output() @staticmethod - def BytesToString(buf): + def BytesToString(buf) -> str: # The default value for optional strings is None # So, we handle it. if buf is None: diff --git a/octoeverywhere/websocketimpl.py b/octoeverywhere/websocketimpl.py index 8c660e6..97b5088 100644 --- a/octoeverywhere/websocketimpl.py +++ b/octoeverywhere/websocketimpl.py @@ -8,7 +8,7 @@ # This class gives a bit of an abstraction over the normal ws class Client: - def __init__(self, url, onWsOpen = None, onWsMsg = None, onWsData = None, onWsClose = None, onWsError = None, headers = None): + def __init__(self, url, onWsOpen = None, onWsMsg = None, onWsData = None, onWsClose = None, onWsError = None, headers:dict = None, subProtocolList:list = None): # Since we also fire onWsError if there is a send error, we need to capture # the callback and have some vars to ensure it only gets fired once. @@ -53,7 +53,8 @@ def OnError(ws, exception): on_close = OnClosed, on_error = OnError, on_data = OnData, - header = headers + header = headers, + subprotocols = subProtocolList ) From 855ec8d4729ae38c8a95333ebc789a69dd69184d Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Mon, 18 Dec 2023 10:40:57 -0800 Subject: [PATCH 002/328] Adding a little logic to help debug a websocket PY lib issue. --- octoeverywhere/octoservercon.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/octoeverywhere/octoservercon.py b/octoeverywhere/octoservercon.py index 9730540..c4ede0f 100644 --- a/octoeverywhere/octoservercon.py +++ b/octoeverywhere/octoservercon.py @@ -7,6 +7,7 @@ from .octosessionimpl import OctoSession from .repeattimer import RepeatTimer from .octopingpong import OctoPingPong +from .threaddebug import ThreadDebug # # This class is responsible for connecting and maintaining a connection to a server. @@ -239,6 +240,8 @@ def Disconnect(self): Sentry.Exception("Exception when calling CloseAllWebStreamsAndDisable from Disconnect.", e) else: self.Logger.info("OctoServerCon Disconnect was called, but we are skipping the CloseAllWebStreamsAndDisable because it has already been done.") + # TODO - Remove this after we figure out this websocket lib dead lock bug. + ThreadDebug.DoThreadDumpLogout(self.Logger) # On every disconnect call, try to disconnect the websocket. We do this because we have seen that for some reason calling Close doesn't seem # to always actually cause the websocket to close and cause RunUntilClosed to return. Thus we hope if we keep trying to close it, maybe it will. From d7a963632f11f08004eb16097ddb87122360e19d Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Mon, 18 Dec 2023 11:11:15 -0800 Subject: [PATCH 003/328] Updating our python packages to stay current and prevent security issues! --- requirements.txt | 12 ++++++------ setup.py | 12 +++++++++++- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/requirements.txt b/requirements.txt index 506a399..7601a29 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,14 +7,14 @@ # # For comments on package lock versions, see the comments in the setup.py file. # -websocket_client>=1.6.0,<1.6.99 -requests>=2.24.0 +websocket_client>=1.7.0,<1.7.99 +requests>=2.31.0 octoflatbuffers==2.0.5 pillow -certifi>=2023.7.22 +certifi>=2023.11.17 rsa>=4.9 -dnspython>=2.3.0 -httpx==0.24.0 -urllib3>=1.26.15,<1.27.0 +dnspython>=2.4.0 +httpx>=0.25.0,<0.26.0 +urllib3>=2.1.0 # The following are required only for Moonraker configparser \ No newline at end of file diff --git a/setup.py b/setup.py index 934233f..179c388 100644 --- a/setup.py +++ b/setup.py @@ -68,7 +68,17 @@ # urllib3 - There is a bug with parsing headers in versions older than 1.26.? (https://github.com/diyan/pywinrm/issues/269). At least 1.26.6 fixes it, ubt we decide to just stick with a newer version. # # Note! These also need to stay in sync with requirements.txt, for the most part they should be the exact same! -plugin_requires = ["websocket_client>=1.6.0,<1.6.99", "requests>=2.24.0", "octoflatbuffers==2.0.5", "pillow", "certifi>=2023.7.22", "rsa>=4.9", "dnspython>=2.3.0", "httpx==0.24.0", "urllib3>=1.26.15,<1.27.0" ] +plugin_requires = [ + "websocket_client>=1.7.0,<1.7.99", + "requests>=2.31.0", + "octoflatbuffers==2.0.5", + "pillow", + "certifi>=2023.11.17", + "rsa>=4.9", + "dnspython>=2.4.0", + "httpx>=0.25.0,<0.26.0", + "urllib3>=2.1.0" + ] ### -------------------------------------------------------------------------------------------------------------------- ### More advanced options that you usually shouldn't have to touch follow after this point From a658ca0958bfd18e36a938ec8b362edb9cdfdb81 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Mon, 18 Dec 2023 11:11:37 -0800 Subject: [PATCH 004/328] Version bump. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 179c388..1648d8f 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this is also parsed by the moonraker module to pull the version, so the string and format must remain the same! -plugin_version = "2.10.1" +plugin_version = "2.10.5" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 33586facaf5d425a0a3b2903c38e401494ce7143 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Mon, 18 Dec 2023 11:53:11 -0800 Subject: [PATCH 005/328] Fixing some package update deps for the sonic pad. --- install.sh | 4 +++- requirements.txt | 4 ++-- setup.py | 6 ++++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/install.sh b/install.sh index 31d0b14..b803c21 100755 --- a/install.sh +++ b/install.sh @@ -219,11 +219,13 @@ install_or_update_system_dependencies() # Note that since cloudflare will auto force http -> https, we use https, but ignore cert errors, that could be # caused by an incorrect date. # Note some companion systems don't have curl installed, so this will fail. + log_info "Ensuring the system date and time is correct..." sudo date -s `curl --insecure 'https://octoeverywhere.com/api/util/date' 2>/dev/null` || true # These we require to be installed in the OS. # Note we need to do this before we create our virtual environment - sudo apt update + log_info "Installing required system packages..." + sudo apt update 1>/dev/null` 2>/dev/null` || true sudo apt install --yes ${PKGLIST} # The PY lib Pillow depends on some system packages that change names depending on the OS. diff --git a/requirements.txt b/requirements.txt index 7601a29..6a8a956 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,13 +7,13 @@ # # For comments on package lock versions, see the comments in the setup.py file. # -websocket_client>=1.7.0,<1.7.99 +websocket_client>=1.6.0,<1.7.99 requests>=2.31.0 octoflatbuffers==2.0.5 pillow certifi>=2023.11.17 rsa>=4.9 -dnspython>=2.4.0 +dnspython>=2.3.0 httpx>=0.25.0,<0.26.0 urllib3>=2.1.0 # The following are required only for Moonraker diff --git a/setup.py b/setup.py index 1648d8f..afa309c 100644 --- a/setup.py +++ b/setup.py @@ -57,8 +57,10 @@ # Version 1.4.0 also has an SSL error in it. https://github.com/websocket-client/websocket-client/issues/857 # Update: We also found a bug where the ping timer doesn't get cleaned up: https://github.com/websocket-client/websocket-client/pull/918 # Thus we need version 1.6.0 or higher. +# The sonic pad runs python 3.7.8 as of 12/18/2023 and websocket_client>=1.7 doesn't support it. So we must keep our version at 1.6 at least for now. # dnspython # We depend on a feature that was released with 2.3.0, so we need to require at least that. +# For the same reason as websocket_client for the sonic pad, we also need to include at least 2.3.0, since 2.3.0 is the last version to support python 3.7.8. # # Other lib version notes: # pillow - We don't require a version of pillow because we don't want to mess with other plugins and we use basic, long lived APIs.\ @@ -69,13 +71,13 @@ # # Note! These also need to stay in sync with requirements.txt, for the most part they should be the exact same! plugin_requires = [ - "websocket_client>=1.7.0,<1.7.99", + "websocket_client>=1.6.0,<1.7.99", "requests>=2.31.0", "octoflatbuffers==2.0.5", "pillow", "certifi>=2023.11.17", "rsa>=4.9", - "dnspython>=2.4.0", + "dnspython>=2.3.0", "httpx>=0.25.0,<0.26.0", "urllib3>=2.1.0" ] From b337b9557c093fe46077f742c99c552806e98d36 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Mon, 18 Dec 2023 11:56:31 -0800 Subject: [PATCH 006/328] One more minor tweak. --- install.sh | 3 +-- requirements.txt | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/install.sh b/install.sh index b803c21..cd6cd4a 100755 --- a/install.sh +++ b/install.sh @@ -212,8 +212,6 @@ install_or_update_system_dependencies() opkg install ${SONIC_PAD_DEP_LIST} pip3 install virtualenv else - log_important "You might be asked for your system password - this is required to install the required system packages." - # It seems a lot of printer control systems don't have the date and time set correctly, and then the fail # getting packages and other downstream things. We will will use our HTTP API to set the current UTC time. # Note that since cloudflare will auto force http -> https, we use https, but ignore cert errors, that could be @@ -224,6 +222,7 @@ install_or_update_system_dependencies() # These we require to be installed in the OS. # Note we need to do this before we create our virtual environment + log_important "You might be asked for your system password - this is required to install the required system packages." log_info "Installing required system packages..." sudo apt update 1>/dev/null` 2>/dev/null` || true sudo apt install --yes ${PKGLIST} diff --git a/requirements.txt b/requirements.txt index 6a8a956..7d089d0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ pillow certifi>=2023.11.17 rsa>=4.9 dnspython>=2.3.0 -httpx>=0.25.0,<0.26.0 +httpx>=0.24.1,<0.26.0 urllib3>=2.1.0 # The following are required only for Moonraker configparser \ No newline at end of file diff --git a/setup.py b/setup.py index afa309c..0d2a9c4 100644 --- a/setup.py +++ b/setup.py @@ -78,7 +78,7 @@ "certifi>=2023.11.17", "rsa>=4.9", "dnspython>=2.3.0", - "httpx>=0.25.0,<0.26.0", + "httpx>=0.24.1,<0.26.0", "urllib3>=2.1.0" ] From 6f159cd4dbc8472dbe7ec4ee60b53198da38a68b Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Mon, 18 Dec 2023 11:59:36 -0800 Subject: [PATCH 007/328] Another one! --- .github/workflows/pylint.yml | 1 + requirements.txt | 2 +- setup.py | 6 ++++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 48939bb..2b34a50 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -7,6 +7,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: + # The sonic pad runs 3.7, so it's important to keep it here to make sure all of our required dependencies work python-version: ["3.7", "3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v2 diff --git a/requirements.txt b/requirements.txt index 7d089d0..426ebf4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,6 +15,6 @@ certifi>=2023.11.17 rsa>=4.9 dnspython>=2.3.0 httpx>=0.24.1,<0.26.0 -urllib3>=2.1.0 +urllib3>=2.0.7 # The following are required only for Moonraker configparser \ No newline at end of file diff --git a/setup.py b/setup.py index 0d2a9c4..9eb99d2 100644 --- a/setup.py +++ b/setup.py @@ -61,13 +61,15 @@ # dnspython # We depend on a feature that was released with 2.3.0, so we need to require at least that. # For the same reason as websocket_client for the sonic pad, we also need to include at least 2.3.0, since 2.3.0 is the last version to support python 3.7.8. +# urllib3 +# There is a bug with parsing headers in versions older than 1.26.? (https://github.com/diyan/pywinrm/issues/269). At least 1.26.6 fixes it, ubt we decide to just stick with a newer version. +# Must include > 2.0.7 due to the sonic pad being on python 3.7.8. # # Other lib version notes: # pillow - We don't require a version of pillow because we don't want to mess with other plugins and we use basic, long lived APIs.\ # certifi - We use to keep certs on the device that we need for let's encrypt. So we want to keep it fresh. # rsa - OctoPrint 1.5.3 requires RAS>=4.0, so we must leave it at 4.0. # httpx - Is an asyncio http lib. It seems to be required by dnspython, but dnspython doesn't enforce it. We had a user having an issue that updated to 0.24.0, and it resolved the issue. -# urllib3 - There is a bug with parsing headers in versions older than 1.26.? (https://github.com/diyan/pywinrm/issues/269). At least 1.26.6 fixes it, ubt we decide to just stick with a newer version. # # Note! These also need to stay in sync with requirements.txt, for the most part they should be the exact same! plugin_requires = [ @@ -79,7 +81,7 @@ "rsa>=4.9", "dnspython>=2.3.0", "httpx>=0.24.1,<0.26.0", - "urllib3>=2.1.0" + "urllib3>=2.0.7" ] ### -------------------------------------------------------------------------------------------------------------------- From 16905aa3fb96a705f2a14bb02fa94eb011bb25f1 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Mon, 18 Dec 2023 12:03:50 -0800 Subject: [PATCH 008/328] One more tweak for the sonic pad. --- requirements.txt | 2 +- setup.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 426ebf4..e14cca4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,6 +15,6 @@ certifi>=2023.11.17 rsa>=4.9 dnspython>=2.3.0 httpx>=0.24.1,<0.26.0 -urllib3>=2.0.7 +urllib3>=1.26.18<2.0.0 # The following are required only for Moonraker configparser \ No newline at end of file diff --git a/setup.py b/setup.py index 9eb99d2..efd2a5f 100644 --- a/setup.py +++ b/setup.py @@ -63,7 +63,7 @@ # For the same reason as websocket_client for the sonic pad, we also need to include at least 2.3.0, since 2.3.0 is the last version to support python 3.7.8. # urllib3 # There is a bug with parsing headers in versions older than 1.26.? (https://github.com/diyan/pywinrm/issues/269). At least 1.26.6 fixes it, ubt we decide to just stick with a newer version. -# Must include > 2.0.7 due to the sonic pad being on python 3.7.8. +# This must be less than 2.0.0, because 2.0.0 requires open ssl 1.1.1+, which the sonic pad doesn't have. # # Other lib version notes: # pillow - We don't require a version of pillow because we don't want to mess with other plugins and we use basic, long lived APIs.\ @@ -81,7 +81,7 @@ "rsa>=4.9", "dnspython>=2.3.0", "httpx>=0.24.1,<0.26.0", - "urllib3>=2.0.7" + "urllib3>=1.26.18<2.0.0" ] ### -------------------------------------------------------------------------------------------------------------------- From 4489881a70d3117ab3f7c6243dd4299963bb29b8 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Mon, 18 Dec 2023 12:11:23 -0800 Subject: [PATCH 009/328] Package version error. --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index e14cca4..cbdc41f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,6 +15,6 @@ certifi>=2023.11.17 rsa>=4.9 dnspython>=2.3.0 httpx>=0.24.1,<0.26.0 -urllib3>=1.26.18<2.0.0 +urllib3>=1.26.15,<2.0.0 # The following are required only for Moonraker configparser \ No newline at end of file diff --git a/setup.py b/setup.py index efd2a5f..e824fa4 100644 --- a/setup.py +++ b/setup.py @@ -81,7 +81,7 @@ "rsa>=4.9", "dnspython>=2.3.0", "httpx>=0.24.1,<0.26.0", - "urllib3>=1.26.18<2.0.0" + "urllib3>=1.26.18,<2.0.0" ] ### -------------------------------------------------------------------------------------------------------------------- From c3a7522876ba5048d418d39bf3d65d23ebaed9a8 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Mon, 15 Jan 2024 08:17:35 -0800 Subject: [PATCH 010/328] Minor update to the installer script. --- install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install.sh b/install.sh index cd6cd4a..97a4364 100755 --- a/install.sh +++ b/install.sh @@ -185,10 +185,10 @@ ensure_py_venv() if [[ $IS_K1_OS -eq 1 ]] then # The K1 requires we setup the virtualenv like this. - python3 /usr/lib/python3.8/site-packages/virtualenv.py -p /usr/bin/python3 --system-site-packages "${OE_ENV}" + python3 /usr/lib/python3.8/site-packages/virtualenv.py -p /usr/bin/python3 "${OE_ENV}" else # Everything else can use this more modern style command. - virtualenv -p /usr/bin/python3 --system-site-packages "${OE_ENV}" + virtualenv -p /usr/bin/python3 "${OE_ENV}" fi } From 4a1f9f6bd0541707430f435e6ee44c329b5dc916 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Tue, 16 Jan 2024 17:03:12 -0800 Subject: [PATCH 011/328] Fixing a k1 installer issue. --- install.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 97a4364..e63bcf7 100755 --- a/install.sh +++ b/install.sh @@ -185,9 +185,11 @@ ensure_py_venv() if [[ $IS_K1_OS -eq 1 ]] then # The K1 requires we setup the virtualenv like this. - python3 /usr/lib/python3.8/site-packages/virtualenv.py -p /usr/bin/python3 "${OE_ENV}" + # --system-site-packages is important for the K1, since it doesn't have much disk space. + python3 /usr/lib/python3.8/site-packages/virtualenv.py -p /usr/bin/python3 --system-site-packages "${OE_ENV}" else # Everything else can use this more modern style command. + # We don't want to use --system-site-packages, so we don't consume whatever packages are on the system. virtualenv -p /usr/bin/python3 "${OE_ENV}" fi } From 0cdec3e8237b3c3dd32cb36f2b534ce5cc7f36c5 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sun, 21 Jan 2024 09:03:28 -0800 Subject: [PATCH 012/328] Fixing a minor bug. --- octoeverywhere/octopingpong.py | 1 + octoeverywhere/octoservercon.py | 147 ++++++++++++++++---------------- octoeverywhere/websocketimpl.py | 9 +- setup.py | 2 +- 4 files changed, 80 insertions(+), 79 deletions(-) diff --git a/octoeverywhere/octopingpong.py b/octoeverywhere/octopingpong.py index 8d805f3..d7de3ac 100644 --- a/octoeverywhere/octopingpong.py +++ b/octoeverywhere/octopingpong.py @@ -133,6 +133,7 @@ def _WorkerThread(self): callback = self.PluginFirstRunLatencyCompleteCallback if callback is not None: self.PluginFirstRunLatencyCompleteCallback() + callback = None except Exception as e: Sentry.Exception("Exception in OctoPingPong thread.", e) diff --git a/octoeverywhere/octoservercon.py b/octoeverywhere/octoservercon.py index c4ede0f..b2f2b2b 100644 --- a/octoeverywhere/octoservercon.py +++ b/octoeverywhere/octoservercon.py @@ -92,16 +92,6 @@ def __init__(self, host, endpoint, isPrimaryConnection, shouldUseLowestLatencySe self.CreationTime = datetime.now() self.LastUserActivityTime = self.CreationTime - # Start the RunFor time checker. - self.RunForTimeChecker = RepeatTimer(self.Logger, self.RunForTimeCheckerIntervalSec, self.OnRunForTimerCallback) - self.RunForTimeChecker.start() - - - def Cleanup(self): - # Stop the RunFor time checker if we have one. - if self.RunForTimeChecker is not None: - self.RunForTimeChecker.Stop() - # Returns a printable string that says the endpoint and the active session id. def GetConnectionString(self): @@ -299,70 +289,79 @@ def OnFirstRunLatencyDataComplete(self): def RunBlocking(self): - while 1: - # Since we want to run forever, we want to make sure any exceptions get caught but then we try again. - try: - # Clear the disconnecting flag. - # We do this just before connects, because this flag weeds out all of the error noise - # that might happen while we are performing a disconnect. But at this time, all of that should be - # 100% done now. - self.IsDisconnecting = False - - # Set the connecting flag, so we know if we are in the middle of a ws connect. - # This is set to false when the websocket is established. - self.IsWsConnecting = True - - # Since there can be old pending actions from old sessions (session == one websocket connection). - # We will keep track of the current session, so old errors from sessions don't effect the new one. - self.ActiveSessionId += 1 - - # Get the new endpoint. This will either be the default endpoint or the lowest latency endpoint. - endpoint = self.GetEndpoint() - - # Connect to the service. - self.Ws = Client(endpoint, self.OnOpened, self.OnMsg, None, self.OnClosed, self.OnError) - self.Logger.info("Attempting to talk to OctoEverywhere, server con "+self.GetConnectionString() + " wsId:"+self.GetWsId(self.Ws)) - self.Ws.RunUntilClosed() - - # Handle disconnects - self.Logger.info("Disconnected from OctoEverywhere, server con "+self.GetConnectionString()) - - # Ensure all proxy sockets are closed. - if self.OctoSession: - self.OctoSession.CloseAllWebStreamsAndDisable() - - except Exception as e: - self.TempDisableLowestLatencyEndpoint = True - Sentry.Exception("Exception in OctoEverywhere's main RunBlocking function. server con:"+self.GetConnectionString()+".", e) - time.sleep(20) - - # On each disconnect, check if the RunFor time is now done. - if self.IsRunForTimeComplete(): - # If our run for time expired, cleanup and return. - self.Cleanup() - self.Logger.info("Server con "+self.GetConnectionString()+" RunFor is complete, disconnected, and exiting the main thread.") - # Exit the main run blocking loop. - return - - # We have a back off time, but always add some random noise as well so not all clients try to use the exact same time. - # Note this applies to all reconnects, even for errors in the system and not server connection loss. - self.WsConnectBackOffSec += random.randint(self.WsConnectRandomMinSec, self.WsConnectRandomMaxSec) - - # Don't sleep if we want to NoWaitReconnect - if self.NoWaitReconnect: - self.NoWaitReconnect = False - self.Logger.info("Skipping reconnect delay due to instant reconnect request.") - else: - self.Logger.info("Sleeping for " + str(self.WsConnectBackOffSec) + " seconds before trying again.") - time.sleep(self.WsConnectBackOffSec) - - # Increment the back off time. - self.WsConnectBackOffSec *= 2 - if self.WsConnectBackOffSec > 180 : - self.WsConnectBackOffSec = 180 - # If we have failed and are waiting over 3 minutes, we will return which will check the server - # protocol again, since it might have changed. - return + runForTimeChecker = None + try: + # Start the RunFor time checker. + # This will always be stopped in the finally before we exit this function. + runForTimeChecker = RepeatTimer(self.Logger, self.RunForTimeCheckerIntervalSec, self.OnRunForTimerCallback) + runForTimeChecker.start() + + while 1: + # Since we want to run forever, we want to make sure any exceptions get caught but then we try again. + try: + # Clear the disconnecting flag. + # We do this just before connects, because this flag weeds out all of the error noise + # that might happen while we are performing a disconnect. But at this time, all of that should be + # 100% done now. + self.IsDisconnecting = False + + # Set the connecting flag, so we know if we are in the middle of a ws connect. + # This is set to false when the websocket is established. + self.IsWsConnecting = True + + # Since there can be old pending actions from old sessions (session == one websocket connection). + # We will keep track of the current session, so old errors from sessions don't effect the new one. + self.ActiveSessionId += 1 + + # Get the new endpoint. This will either be the default endpoint or the lowest latency endpoint. + endpoint = self.GetEndpoint() + + # Connect to the service. + self.Ws = Client(endpoint, self.OnOpened, self.OnMsg, None, self.OnClosed, self.OnError) + self.Logger.info("Attempting to talk to OctoEverywhere, server con "+self.GetConnectionString() + " wsId:"+self.GetWsId(self.Ws)) + self.Ws.RunUntilClosed() + + # Handle disconnects + self.Logger.info("Disconnected from OctoEverywhere, server con "+self.GetConnectionString()) + + # Ensure all proxy sockets are closed. + if self.OctoSession: + self.OctoSession.CloseAllWebStreamsAndDisable() + + except Exception as e: + self.TempDisableLowestLatencyEndpoint = True + Sentry.Exception("Exception in OctoEverywhere's main RunBlocking function. server con:"+self.GetConnectionString()+".", e) + time.sleep(20) + + # On each disconnect, check if the RunFor time is now done. + if self.IsRunForTimeComplete(): + self.Logger.info("Server con "+self.GetConnectionString()+" RunFor is complete, disconnected, and exiting the main thread.") + # Exit the main run blocking loop. + return + + # We have a back off time, but always add some random noise as well so not all clients try to use the exact same time. + # Note this applies to all reconnects, even for errors in the system and not server connection loss. + self.WsConnectBackOffSec += random.randint(self.WsConnectRandomMinSec, self.WsConnectRandomMaxSec) + + # Don't sleep if we want to NoWaitReconnect + if self.NoWaitReconnect: + self.NoWaitReconnect = False + self.Logger.info("Skipping reconnect delay due to instant reconnect request.") + else: + self.Logger.info("Sleeping for " + str(self.WsConnectBackOffSec) + " seconds before trying again.") + time.sleep(self.WsConnectBackOffSec) + + # Increment the back off time. + self.WsConnectBackOffSec *= 2 + if self.WsConnectBackOffSec > 180 : + self.WsConnectBackOffSec = 180 + # If we have failed and are waiting over 3 minutes, we will return which will check the server + # protocol again, since it might have changed. + return + finally: + # Before we exit this function, we need to always stop the repeat timer we started. + if runForTimeChecker is not None: + runForTimeChecker.Stop() def SendMsg(self, msgBytes): diff --git a/octoeverywhere/websocketimpl.py b/octoeverywhere/websocketimpl.py index 97b5088..56ea3ee 100644 --- a/octoeverywhere/websocketimpl.py +++ b/octoeverywhere/websocketimpl.py @@ -91,8 +91,9 @@ def fireWsErrorCallbackThread(self, exception): # ignore any exceptions. try: self.Ws.close() - except Exception as _ : - pass + except Exception as ex : + Sentry.Exception("Websocket fireWsErrorCallbackThread close exception", ex) + except Exception as e : Sentry.Exception("Websocket client exception in fireWsErrorCallbackThread", e) @@ -136,8 +137,8 @@ def Close(self): # Always try to call close, even if we have already done it. try: self.Ws.close() - except Exception: - pass + except Exception as e: + Sentry.Exception("Websocket close exception", e) def Send(self, msgBytes, isData): diff --git a/setup.py b/setup.py index e824fa4..4e7a0df 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this is also parsed by the moonraker module to pull the version, so the string and format must remain the same! -plugin_version = "2.10.5" +plugin_version = "2.10.6" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 2502dfeef5c633ca8beb498b72d4426c9bbcc3e4 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Tue, 23 Jan 2024 20:42:06 -0800 Subject: [PATCH 013/328] Minor fix for keeping webcam settings better in sync. --- moonraker_octoeverywhere/moonrakerhost.py | 6 ++++-- setup.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/moonraker_octoeverywhere/moonrakerhost.py b/moonraker_octoeverywhere/moonrakerhost.py index c9b56d5..3dbccfc 100644 --- a/moonraker_octoeverywhere/moonrakerhost.py +++ b/moonraker_octoeverywhere/moonrakerhost.py @@ -299,7 +299,9 @@ def OnPluginUpdateRequired(self): def OnMoonrakerWsOpenAndAuthed(self): # Kick off the webcam settings helper, to ensure it pulls fresh settings if desired. - self.MoonrakerWebcamHelper.KickOffWebcamSettingsUpdate() + # Use force, because the websocket might not open for some time and the first auto get might fail. + # When when moonraker connects, for the settings get, so ensure we are in sync with the system. + self.MoonrakerWebcamHelper.KickOffWebcamSettingsUpdate(forceUpdate=True) # Also allow the database logic to ensure our public keys exist and are updated. self.MoonrakerDatabase.EnsureOctoEverywhereDatabaseEntry() @@ -309,7 +311,7 @@ def OnMoonrakerWsOpenAndAuthed(self): # def OnWebcamSettingsChanged(self): # Set the force flag to true, since we know the settings just changed. - self.MoonrakerWebcamHelper.KickOffWebcamSettingsUpdate(True) + self.MoonrakerWebcamHelper.KickOffWebcamSettingsUpdate(forceUpdate=True) # # MoonrakerClient ConnectionStatusHandler Interface - Called by the MoonrakerClient when the moonraker connection has been established and klippy is fully ready to use. diff --git a/setup.py b/setup.py index 4e7a0df..ce8c2da 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this is also parsed by the moonraker module to pull the version, so the string and format must remain the same! -plugin_version = "2.10.6" +plugin_version = "2.10.7" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From e2022be686090db0e23aae4b76dbaefc14d2fa38 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Mon, 29 Jan 2024 21:27:23 -0800 Subject: [PATCH 014/328] Very small tweak. --- moonraker_installer/Permissions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moonraker_installer/Permissions.py b/moonraker_installer/Permissions.py index 8edf791..5816275 100644 --- a/moonraker_installer/Permissions.py +++ b/moonraker_installer/Permissions.py @@ -34,7 +34,7 @@ def EnsureRunningAsRootOrSudo(self, context:Context) -> None: # For the Sonic Pad and K1 setup, the only user is root, so it's ok. if context.IsObserverSetup is False and context.IsCrealityOs() is False: if context.UserName.lower() == Permissions.c_RootUserName: - raise Exception("The installer was ran under the root user, this will cause problems with Moonraker. Please run the installer script as a non-root user, usually that's the `pi` user.") + raise Exception("The installer was ran under the root user, this will cause problems with Moonraker. Please run the installer script as a non-root user, usually that's the `pi` user or 'mks' for MKS PI.") # But regardless of the user, we must have sudo permissions. # pylint: disable=no-member # Linux only From 149f1e111f12b6d0f3225f1addf210cf03bedf8c Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Tue, 30 Jan 2024 21:07:11 -0800 Subject: [PATCH 015/328] Adding time sync logic to ensure that timesyncd is installed and enabled on the device. If not, the clock can get out of sync, which will case SSL handshakes to fail. --- .vscode/settings.json | 2 + moonraker_installer/Installer.py | 6 +++ moonraker_installer/TimeSync.py | 70 ++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+) create mode 100644 moonraker_installer/TimeSync.py diff --git a/.vscode/settings.json b/.vscode/settings.json index 33b4dfc..c9d2dc1 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -168,7 +168,9 @@ "telemetryaccumulator", "thirdlayerdone", "threaddebug", + "timedatectl", "timerprogress", + "timesyncd", "toradio", "Trustpilot", "UDISK", diff --git a/moonraker_installer/Installer.py b/moonraker_installer/Installer.py index 1bb61c1..3368bd2 100644 --- a/moonraker_installer/Installer.py +++ b/moonraker_installer/Installer.py @@ -10,6 +10,7 @@ from .Configure import Configure from .Updater import Updater from .Permissions import Permissions +from .TimeSync import TimeSync from .Frontend import Frontend from .Uninstall import Uninstall @@ -85,6 +86,11 @@ def _RunInternal(self): self.PrintHelp() return + # Ensure that the system clock sync is enabled. For some MKS PI systems the OS time is wrong and sync is disabled. + # The user would of had to manually correct the time to get this installer running, but we will ensure that the + # time sync systemd service is enabled to keep the clock in sync after reboots, otherwise it will cause SSL errors. + TimeSync.EnsureNtpSyncEnabled() + # Ensure the script at least has sudo permissions. # It's required to set file permission and to write / restart the service. # See comments in the function for details. diff --git a/moonraker_installer/TimeSync.py b/moonraker_installer/TimeSync.py new file mode 100644 index 0000000..6bb4d5c --- /dev/null +++ b/moonraker_installer/TimeSync.py @@ -0,0 +1,70 @@ +from .Util import Util +from .Logging import Logger + +# This helper class ensures that the system's ntp clock sync service is enabled and active. +# We found some MKS PI systems didn't have it on, and would be years out of sync on reboot. +# This is a problem because SSL will fail if the date is too far out of sync. +# +# For the most part, this class is best effort. It will try to get everything setup, but if it fails, +# we won't stop the setup. +class TimeSync: + + @staticmethod + def EnsureNtpSyncEnabled(): + Logger.Info("Ensuring that time sync is enabled...") + + # Ensure that NTP is uninstalled, since this conflicts with timesyncd + TimeSync._RunSystemCommand("sudo apt -y purge ntp ntpdate ntpsec-ntpdate") + + # Ensure timedatectl is installed. On all most systems it will be already. + TimeSync._RunSystemCommand("sudo apt install -y systemd-timesyncd") + TimeSync._PrintTimeSyncDStatus() + + # Ensure time servers are set in the config file. + TimeSync._UpdateTimeSyncdConfig() + + # Reload and start the systemd service + TimeSync._RunSystemCommand("sudo systemctl daemon-reload") + TimeSync._RunSystemCommand("sudo systemctl enable systemd-timesyncd") + TimeSync._RunSystemCommand("sudo systemctl restart systemd-timesyncd") + TimeSync._RunSystemCommand("sudo timedatectl set-ntp on") + + # Print the status outcome. + TimeSync._PrintTimeSyncDStatus() + + + @staticmethod + def _UpdateTimeSyncdConfig(): + targetFilePath = "/etc/systemd/timesyncd.conf" + try: + # After writing, read the file and insert any comments we have. + outputLines = [] + with open(targetFilePath, 'r', encoding="utf-8") as f: + lines = f.readlines() + for line in lines: + lineLower = line.lower() + if lineLower.startswith("#ntp="): + # This is case sensitive! + outputLines.append("NTP=0.pool.ntp.org 1.pool.ntp.org 2.pool.ntp.org 3.pool.ntp.org\n") + else: + outputLines.append(line) + # This will only happen if we have sudo powers. + with open(targetFilePath, 'w', encoding="utf-8") as f: + f.writelines(outputLines) + except Exception as e: + Logger.Debug(f"TimeSync update config exception. (this is ok) {str(e)}") + + + @staticmethod + def _RunSystemCommand(cmd:str): + (code, stdOut, errOut) = Util.RunShellCommand(cmd, False) + if code == 0: + Logger.Debug(f"TimeSync System Command Success. Cmd: {cmd}") + if code != 0: + Logger.Debug(f"TimeSync System Command FAILED. (this is ok) Cmd: `{cmd}` - `{str(stdOut)}` - `{str(errOut)}`") + + + @staticmethod + def _PrintTimeSyncDStatus(): + (_, stdOut, errOut) = Util.RunShellCommand("sudo timedatectl status", False) + Logger.Debug(f"TimeSync Status:\r\n{stdOut} {errOut}") From a4b24ba9edfdda1875f080cc07308b3d68010df5 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sat, 3 Feb 2024 12:39:02 -0800 Subject: [PATCH 016/328] Adding flag to not print all WS messages when in debug logging. --- moonraker_octoeverywhere/moonrakerclient.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/moonraker_octoeverywhere/moonrakerclient.py b/moonraker_octoeverywhere/moonrakerclient.py index 1cb5ebe..e3bdf40 100644 --- a/moonraker_octoeverywhere/moonrakerclient.py +++ b/moonraker_octoeverywhere/moonrakerclient.py @@ -61,6 +61,8 @@ class MoonrakerClient: # Logic for a static singleton _Instance = None + # If enabled, this prints all of the websocket messages sent and received. + WebSocketMessageDebugging = False @staticmethod def Init(logger, isObserverMode:bool, moonrakerConfigFilePath:str, observerConfigPath:str, printerId, connectionStatusHandler, pluginVersionStr): @@ -299,7 +301,8 @@ def _WebSocketSend(self, jsonStr:str) -> bool: return False # Print for debugging. - self.Logger.debug("Ws ->: "+jsonStr) + if MoonrakerClient.WebSocketMessageDebugging and self.Logger.isEnabledFor(logging.DEBUG): + self.Logger.debug("Ws ->: %s",+jsonStr) # Send under lock. try: @@ -628,7 +631,7 @@ def _onWsMsg(self, ws, msgBytes: bytes): method_CanBeNone = msgObj["method"] # Print for debugging - if self.Logger.isEnabledFor(logging.DEBUG): + if MoonrakerClient.WebSocketMessageDebugging and self.Logger.isEnabledFor(logging.DEBUG): # Exclude this really chatty message. msgStr = msgBytes.decode(encoding="utf-8") if "moonraker_stats" not in msgStr: From dcb771d01c7732bfe03346746e8921d95b8d899e Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sun, 4 Feb 2024 22:16:17 -0800 Subject: [PATCH 017/328] Minor changes to update to moonraker's new updatemanager APIs. --- install.sh | 12 ++++++++++-- moonraker-system-dependencies.json | 10 ++++++++++ moonraker_installer/Configure.py | 2 ++ moonraker_installer/Installer.py | 7 ++++++- moonraker_installer/Logging.py | 13 ++++++++++++- moonraker_octoeverywhere/logger.py | 4 ++-- moonraker_octoeverywhere/systemconfigmanager.py | 9 +++++++++ 7 files changed, 51 insertions(+), 6 deletions(-) create mode 100644 moonraker-system-dependencies.json diff --git a/install.sh b/install.sh index e63bcf7..6715304 100755 --- a/install.sh +++ b/install.sh @@ -69,14 +69,18 @@ OE_ENV="${HOME}/octoeverywhere-env" # For python packages, the `requirements.txt` package is used on update. # This var name MUST BE `PKGLIST`!! # +# Note! This was deprecated in newer versions of moonraker, instead the deps are in the moonraker-system-dependencies.json file. +# For now we will keep both around AND IN SYNC so we can support older versions of moonraker. +# # The python requirements are for the installer and plugin # The virtualenv is for our virtual package env we create # The curl requirement is for some things in this bootstrap script. -PKGLIST="python3 python3-pip virtualenv curl" +# python3-venv is required for teh virtualenv command to fully work. +PKGLIST="python3 python3-pip virtualenv python3-venv curl" # For the Creality OS, we only need to install these. # We don't override the default name, since that's used by the Moonraker installer # Note that we DON'T want to use the same name as above (not even in this comment) because some parsers might find it. -SONIC_PAD_DEP_LIST="python3 python3-pip" +SONIC_PAD_DEP_LIST="python3 python3-pip python3-venv" # @@ -187,6 +191,10 @@ ensure_py_venv() # The K1 requires we setup the virtualenv like this. # --system-site-packages is important for the K1, since it doesn't have much disk space. python3 /usr/lib/python3.8/site-packages/virtualenv.py -p /usr/bin/python3 --system-site-packages "${OE_ENV}" + # For the K1, we can't install `python3-venv` which is needed to fully support the virtualenv command. + # Without it, sometimes the .../bin/activate file doesn't exist, which moonraker expects to exist. + # Luckily moonraker doesn't actually do anything with the activate file, so we can just create it. + touch "${OE_ENV}/bin/activate" else # Everything else can use this more modern style command. # We don't want to use --system-site-packages, so we don't consume whatever packages are on the system. diff --git a/moonraker-system-dependencies.json b/moonraker-system-dependencies.json new file mode 100644 index 0000000..0a48172 --- /dev/null +++ b/moonraker-system-dependencies.json @@ -0,0 +1,10 @@ +{ + "_comment": "These must stay in sync with the install.sh PKGLIST until the deprecated list is removed!", + "debian": [ + "python3", + "python3-pip", + "virtualenv", + "python3-venv", + "curl" + ] +} \ No newline at end of file diff --git a/moonraker_installer/Configure.py b/moonraker_installer/Configure.py index 7418693..11e4fb9 100644 --- a/moonraker_installer/Configure.py +++ b/moonraker_installer/Configure.py @@ -93,6 +93,8 @@ def Run(self, context:Context): elif context.OsType == OsTypes.K1: # For the k1, there's only ever one moonraker and we know the exact service naming convention. # Note we use 66 to ensure we start after moonraker. + # This is page for details on the file name: https://docs.oracle.com/cd/E36784_01/html/E36882/init.d-4.html + # Note the 'S66' string is looked for in the plugin's EnsureUpdateManagerFilesSetup function. So it must not change! context.ServiceName = f"S66{Configure.c_ServiceCommonName}_service" context.ServiceFilePath = os.path.join(Paths.CrealityOsServiceFilePath, context.ServiceName) else: diff --git a/moonraker_installer/Installer.py b/moonraker_installer/Installer.py index 3368bd2..84d353b 100644 --- a/moonraker_installer/Installer.py +++ b/moonraker_installer/Installer.py @@ -4,7 +4,7 @@ from .Linker import Linker from .Logging import Logger from .Service import Service -from .Context import Context +from .Context import Context, OsTypes from .Discovery import Discovery from .DiscoveryObserver import DiscoveryObserver from .Configure import Configure @@ -177,6 +177,11 @@ def _RunInternal(self): Logger.Blank() Logger.Blank() + # At the end on success, for OSs that don't have very much disk space, clean up the installer log file, since it's probably not needed. + # If we need the log file for some reason, we should add a flag to the context to keep it. + if context.OsType == OsTypes.SonicPad or context.OsType == OsTypes.K1: + Logger.DeleteLogFile() + def GetArgumentObjectStr(self) -> str: # We want to skip arguments until we find the json string and then concat all args after that together. diff --git a/moonraker_installer/Logging.py b/moonraker_installer/Logging.py index 1bce82a..03de87d 100644 --- a/moonraker_installer/Logging.py +++ b/moonraker_installer/Logging.py @@ -16,13 +16,15 @@ class Logger: IsDebugEnabled = False OutputFile = None + OutputFilePath = None @staticmethod def InitFile(userHomePath): try: + Logger.OutputFilePath = os.path.join(userHomePath, "octoeverywhere-installer.log") # pylint: disable=consider-using-with - Logger.OutputFile = open(os.path.join(userHomePath, "octoeverywhere-installer.log"), "w", encoding="utf-8") + Logger.OutputFile = open(Logger.OutputFilePath, "w", encoding="utf-8") except Exception as e: print("Failed to make log file. "+str(e)) @@ -36,6 +38,15 @@ def Finalize(): pass + @staticmethod + def DeleteLogFile(): + try: + Logger.Finalize() + os.remove(Logger.OutputFilePath) + except Exception: + pass + + @staticmethod def EnableDebugLogging(): Logger.IsDebugEnabled = True diff --git a/moonraker_octoeverywhere/logger.py b/moonraker_octoeverywhere/logger.py index 6865abb..5058a7e 100644 --- a/moonraker_octoeverywhere/logger.py +++ b/moonraker_octoeverywhere/logger.py @@ -40,8 +40,8 @@ def GetLogger(config, klipperLogDir, logLevelOverride_CanBeNone) -> logging.Logg logger.addHandler(std) # Setup the file logger - maxFileSizeBytes = config.GetIntIfInRange(Config.LoggingSection, Config.LogFileMaxSizeMbKey, 5, 1, 5000) * 1024 * 1024 - maxFileCount = config.GetIntIfInRange(Config.LoggingSection, Config.LogFileMaxCountKey, 3, 1, 50) + maxFileSizeBytes = config.GetIntIfInRange(Config.LoggingSection, Config.LogFileMaxSizeMbKey, 3, 1, 5000) * 1024 * 1024 + maxFileCount = config.GetIntIfInRange(Config.LoggingSection, Config.LogFileMaxCountKey, 1, 1, 50) file = logging.handlers.RotatingFileHandler( os.path.join(klipperLogDir, "octoeverywhere.log"), maxBytes=maxFileSizeBytes, backupCount=maxFileCount) diff --git a/moonraker_octoeverywhere/systemconfigmanager.py b/moonraker_octoeverywhere/systemconfigmanager.py index 514495a..488912d 100644 --- a/moonraker_octoeverywhere/systemconfigmanager.py +++ b/moonraker_octoeverywhere/systemconfigmanager.py @@ -12,6 +12,11 @@ class SystemConfigManager: @staticmethod def EnsureUpdateManagerFilesSetup(logger, klipperConfigDir, serviceName, pyVirtEnvRoot, repoRoot): + # Special case for K1 and K1 max setups. If the service file name is the special init.d name, we can just use + # the started "octoeverywhere" and the update manager will find the right service to manage. + if serviceName.startswith("S66"): + serviceName = "octoeverywhere" + # Create the expected update config contents # Note that the update_manager extension name and the managed_services names must match, and the must match the systemd service file name. d = { @@ -26,8 +31,12 @@ def EnsureUpdateManagerFilesSetup(logger, klipperConfigDir, serviceName, pyVirtE channel: beta path: {RepoRootFolder} origin: https://github.com/QuinnDamerell/OctoPrint-OctoEverywhere.git +# env is deprecated, but we must keep it around for now, for older installs. env: {pyVirtEnvRoot}/bin/python +virtualenv: {pyVirtEnvRoot} requirements: requirements.txt +# system_dependencies is newer and replaces install_script for the list of system deps. But for now we need both for older installs. +system_dependencies: moonraker-system-dependencies.json install_script: install.sh managed_services: {ServiceName} From 1b709d331ff02091e8ab67e55d75feeaebe00519 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sun, 4 Feb 2024 22:33:12 -0800 Subject: [PATCH 018/328] Another minor update to fix a Moonraker installer warning. --- .../systemconfigmanager.py | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/moonraker_octoeverywhere/systemconfigmanager.py b/moonraker_octoeverywhere/systemconfigmanager.py index 488912d..f6c0c77 100644 --- a/moonraker_octoeverywhere/systemconfigmanager.py +++ b/moonraker_octoeverywhere/systemconfigmanager.py @@ -1,5 +1,6 @@ import os import subprocess +import logging class SystemConfigManager: @@ -10,15 +11,30 @@ class SystemConfigManager: # This also write a block that's used to allow the announcement system to show updates from our repo. # This function ensures they exist and are up to date. If not, they are fixed. @staticmethod - def EnsureUpdateManagerFilesSetup(logger, klipperConfigDir, serviceName, pyVirtEnvRoot, repoRoot): + def EnsureUpdateManagerFilesSetup(logger:logging.Logger, klipperConfigDir, serviceName, pyVirtEnvRoot, repoRoot): # Special case for K1 and K1 max setups. If the service file name is the special init.d name, we can just use # the started "octoeverywhere" and the update manager will find the right service to manage. if serviceName.startswith("S66"): serviceName = "octoeverywhere" + # Some setups, (it seems mostly like on the K1 and K1 max) don't fully setup the virtual env, but it's setup enough things work. + # However the Moonraker update manager checks for ./bin/activate to be there and it must be a file. Luckily it doesn't use the activate script, it only uses + # the python and pip executables. So we can make an dummy file to make Moonraker happy. + try: + activateFilePath = os.path.join(pyVirtEnvRoot, "bin", "activate") + if os.path.exists(activateFilePath) is False: + logger.warn("No virtual env active script was found, we are creating a dummy file.") + with open(activateFilePath, "w", encoding="utf-8") as file: + file.write("echo 'This is a dummy file created by the OctoEverywhere plugin to make Moonraker happy.'") + except Exception as e: + logger.error("Failed to create the virtual env dummy activate file. "+str(e)) + # Create the expected update config contents # Note that the update_manager extension name and the managed_services names must match, and the must match the systemd service file name. + # + # Note about deprecated options. We have the new option but it's commented out, because if we include both the new and old options moonraker warns about unparsed vars (the old options) + # So for now, we use the old ones, until moonraker drops support for them and we have to move. The problem is then the plugin will not work on older installs after that. d = { 'RepoRootFolder': repoRoot, 'ServiceName' : serviceName, @@ -31,12 +47,12 @@ def EnsureUpdateManagerFilesSetup(logger, klipperConfigDir, serviceName, pyVirtE channel: beta path: {RepoRootFolder} origin: https://github.com/QuinnDamerell/OctoPrint-OctoEverywhere.git -# env is deprecated, but we must keep it around for now, for older installs. +# env is deprecated for virtualenv, but for now we can only use one and must use the older option for compat. env: {pyVirtEnvRoot}/bin/python -virtualenv: {pyVirtEnvRoot} +#virtualenv: {pyVirtEnvRoot} +# requirements is deprecated for system_dependencies, but for now we can only use one and must use the older option for compat. requirements: requirements.txt -# system_dependencies is newer and replaces install_script for the list of system deps. But for now we need both for older installs. -system_dependencies: moonraker-system-dependencies.json +# system_dependencies: moonraker-system-dependencies.json install_script: install.sh managed_services: {ServiceName} From dbbfcc8fa027eb2181414f0069f26528550a5854 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sun, 4 Feb 2024 22:39:57 -0800 Subject: [PATCH 019/328] Version bump. --- moonraker_octoeverywhere/systemconfigmanager.py | 1 + setup.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/moonraker_octoeverywhere/systemconfigmanager.py b/moonraker_octoeverywhere/systemconfigmanager.py index f6c0c77..8097e56 100644 --- a/moonraker_octoeverywhere/systemconfigmanager.py +++ b/moonraker_octoeverywhere/systemconfigmanager.py @@ -15,6 +15,7 @@ def EnsureUpdateManagerFilesSetup(logger:logging.Logger, klipperConfigDir, servi # Special case for K1 and K1 max setups. If the service file name is the special init.d name, we can just use # the started "octoeverywhere" and the update manager will find the right service to manage. + # This was tested, and both the UI to control services and the update manager worked with this setup. if serviceName.startswith("S66"): serviceName = "octoeverywhere" diff --git a/setup.py b/setup.py index ce8c2da..e5ac04c 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this is also parsed by the moonraker module to pull the version, so the string and format must remain the same! -plugin_version = "2.10.7" +plugin_version = "2.10.8" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 7250fb21123a0094694c5f062317e21c93134ef2 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sun, 4 Feb 2024 22:50:35 -0800 Subject: [PATCH 020/328] A little cleanup. --- install.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/install.sh b/install.sh index 6715304..0fc5075 100755 --- a/install.sh +++ b/install.sh @@ -191,10 +191,6 @@ ensure_py_venv() # The K1 requires we setup the virtualenv like this. # --system-site-packages is important for the K1, since it doesn't have much disk space. python3 /usr/lib/python3.8/site-packages/virtualenv.py -p /usr/bin/python3 --system-site-packages "${OE_ENV}" - # For the K1, we can't install `python3-venv` which is needed to fully support the virtualenv command. - # Without it, sometimes the .../bin/activate file doesn't exist, which moonraker expects to exist. - # Luckily moonraker doesn't actually do anything with the activate file, so we can just create it. - touch "${OE_ENV}/bin/activate" else # Everything else can use this more modern style command. # We don't want to use --system-site-packages, so we don't consume whatever packages are on the system. From e7d67372995af6725381e2dcf29f13add06c0d2c Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sun, 4 Feb 2024 22:51:59 -0800 Subject: [PATCH 021/328] Minor logging cleanup. --- octoeverywhere/octosessionimpl.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/octoeverywhere/octosessionimpl.py b/octoeverywhere/octosessionimpl.py index d7eecc5..065ff1c 100644 --- a/octoeverywhere/octosessionimpl.py +++ b/octoeverywhere/octosessionimpl.py @@ -172,7 +172,10 @@ def HandleWebStreamMessage(self, msg): if webStreamMsg.IsOpenMsg() is False: # TODO - Handle messages that arrive for just closed streams better. isCloseMessage = webStreamMsg.IsCloseMsg() - self.Logger.warn("We got a web stream message for a stream id [" + str(streamId) + "] that doesn't exist and isn't an open message. IsClose:"+str(isCloseMessage)) + if isCloseMessage: + self.Logger.debug("We got a web stream message for a stream id [" + str(streamId) + "] that doesn't exist and isn't an open message. IsClose:"+str(isCloseMessage)) + else: + self.Logger.warn("We got a web stream message for a stream id [" + str(streamId) + "] that doesn't exist and isn't an open message. IsClose:"+str(isCloseMessage)) # Don't throw, because this message maybe be coming in from the server as the local side closed. return From c19269e157601c8386b7926f6566c41db7be1c23 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Tue, 6 Feb 2024 21:58:29 -0800 Subject: [PATCH 022/328] Minor fix for the sonic pad. --- install.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 0fc5075..7807158 100755 --- a/install.sh +++ b/install.sh @@ -80,7 +80,8 @@ PKGLIST="python3 python3-pip virtualenv python3-venv curl" # For the Creality OS, we only need to install these. # We don't override the default name, since that's used by the Moonraker installer # Note that we DON'T want to use the same name as above (not even in this comment) because some parsers might find it. -SONIC_PAD_DEP_LIST="python3 python3-pip python3-venv" +# Note we exclude virtualenv python3-venv curl because they can't be installed on the sonic pad via the package manager. +SONIC_PAD_DEP_LIST="python3 python3-pip" # From fcf511253545bab356886519becb286f4959f240 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sat, 17 Feb 2024 11:13:08 -0800 Subject: [PATCH 023/328] Porting some layer notification fixes that the OctoApp developer made! --- .vscode/settings.json | 1 + octoeverywhere/notificationshandler.py | 39 ++++++++++++++++++++++---- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index c9d2dc1..48fd2b2 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -198,6 +198,7 @@ "websockets", "webstream", "zchange", + "zhop", "zhops", "zmoves", "zoffset", diff --git a/octoeverywhere/notificationshandler.py b/octoeverywhere/notificationshandler.py index 05d75a0..c547921 100644 --- a/octoeverywhere/notificationshandler.py +++ b/octoeverywhere/notificationshandler.py @@ -74,6 +74,8 @@ def __init__(self, logger:logging.Logger, printerStateInterface): self.zOffsetNotAtLowestCount = 0 self.zOffsetHasSeenPositiveExtrude = False self.zOffsetTrackingStartTimeSec = 0.0 + self.FirstLayerDoneSince = 0.0 + self.ThirdLayerDoneSince = 0.0 self.ProgressCompletionReported = [] self.PrintId = "none" self.PrintStartTimeSec = 0 @@ -96,6 +98,8 @@ def ResetForNewPrint(self, restoreDurationOffsetSec_OrNone): self.PingTimerHoursReported = 0 self.HasSendFirstLayerDoneMessage = False self.HasSendThirdLayerDoneMessage = False + self.FirstLayerDoneSince = 0.0 + self.ThirdLayerDoneSince = 0.0 # The following values are used to figure out when the first layer is done. self.zOffsetLowestSeenMM = 1337.0 self.zOffsetNotAtLowestCount = 0 @@ -514,14 +518,39 @@ def _OnFirstLayerWatchTimer(self): if currentLayer is not None and totalLayers is not None: # We have layer info from the system, use this to handle the events. - # If we are over the first layer and haven't sent the notification, do it now. + # If we are over the first layer and haven't sent the notification, start the timer. + # We use this time to make sure that the print is still in the first layer complete state and it's not a zhop or something. if currentLayer > 1 and self.HasSendFirstLayerDoneMessage is False: - self.HasSendFirstLayerDoneMessage = True - self._sendEvent("firstlayerdone") + if self.FirstLayerDoneSince < 0.1: + self.Logger.debug("First Layer Logic - Starting delay timer.") + self.FirstLayerDoneSince = time.time() + elif time.time() - self.FirstLayerDoneSince < 10.0: + self.Logger.debug("First Layer Logic - Waiting delay time to expire.") + else: + self.Logger.debug("First Layer Logic - Done.") + self.HasSendFirstLayerDoneMessage = True + self._sendEvent("firstlayerdone") + + # If we fall out of the delay timer wait, reset the timer. + if currentLayer <= 1 and self.FirstLayerDoneSince > 0.0: + self.Logger.debug("First Layer Logic - Reset.") + self.FirstLayerDoneSince = 0.0 + # If we are past the 3rd, layer, do the same. if currentLayer > 3 and self.HasSendThirdLayerDoneMessage is False: - self.HasSendThirdLayerDoneMessage = True - self._sendEvent("thirdlayerdone") + if self.ThirdLayerDoneSince < 0.1: + self.Logger.debug( "Third Layer Logic - Starting delay timer.") + self.ThirdLayerDoneSince = time.time() + elif time.time() - self.ThirdLayerDoneSince < 10.0: + self.Logger.debug( "Third Layer Logic - Waiting delay time to expire.") + else: + self.Logger.debug( "Third Layer Logic - Done.") + self.HasSendThirdLayerDoneMessage = True + self._sendEvent("thirdlayerdone") + + if currentLayer <= 3 and self.ThirdLayerDoneSince > 0.0: + self.Logger.debug("Third Layer Logic - Reset.") + self.ThirdLayerDoneSince = 0.0 # If we return true, the time will continue, otherwise it will stop. isDone = self.HasSendFirstLayerDoneMessage is True and self.HasSendThirdLayerDoneMessage is True From 90bc703e07440994e72184078e441251fca6dec6 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Mon, 19 Feb 2024 17:53:18 -0800 Subject: [PATCH 024/328] Adding better webcam config logic for plugins that report can snapshot incorrectly. --- octoprint_octoeverywhere/octoprintwebcamhelper.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/octoprint_octoeverywhere/octoprintwebcamhelper.py b/octoprint_octoeverywhere/octoprintwebcamhelper.py index ff563d4..d768ca3 100644 --- a/octoprint_octoeverywhere/octoprintwebcamhelper.py +++ b/octoprint_octoeverywhere/octoprintwebcamhelper.py @@ -71,11 +71,12 @@ def GetWebcamConfig(self): # Log for debugging. if self.Logger.isEnabledFor(logging.DEBUG): self.Logger.debug(f"OctoPrint Webcam Config Found: Name: {webcamName}, Can Snapshot: {webcam.canSnapshot}, Webcam Snapshot: \"{webcam.snapshotDisplay}\", Extras: {json.dumps(webcam.extras)}") + + # Some times this bool seems to be reported incorrectly so for now we don't skip the camera if it's set. # Since the snapshot is critical for Gadget and others, only allow webcams that have snapshot (for now) # Also note the webcam system has a fallback for stream url only webcams, we could rely on that? if webcam.canSnapshot is False: - self.Logger.info(f"We found a webcam {webcamName} but it doesn't support snapshots, so we are ignoring it.") - continue + self.Logger.info(f"We found a webcam {webcamName} but it doesn't support snapshots, we will try to detect the snapshot URL for ourselves.") # Make an empty webcam settings item to fill. webSettingsItem = WebcamSettingItem(webcamName) From a93296b49498cc644d8e760d5740d356a88beb64 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Mon, 19 Feb 2024 17:54:28 -0800 Subject: [PATCH 025/328] Version bump. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e5ac04c..bcc418a 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this is also parsed by the moonraker module to pull the version, so the string and format must remain the same! -plugin_version = "2.10.8" +plugin_version = "2.10.9" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From fdf1bd6e14dd36b36fbbe407ee88aab3067baf95 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Wed, 21 Feb 2024 20:56:07 -0800 Subject: [PATCH 026/328] Minor fixes for the Klipper installer. --- moonraker_installer/Frontend.py | 10 ++++++++-- moonraker_installer/Installer.py | 2 +- moonraker_installer/Logging.py | 12 +++++++++++- moonraker_installer/Updater.py | 5 +++++ 4 files changed, 25 insertions(+), 4 deletions(-) diff --git a/moonraker_installer/Frontend.py b/moonraker_installer/Frontend.py index c6ff624..6d938d7 100644 --- a/moonraker_installer/Frontend.py +++ b/moonraker_installer/Frontend.py @@ -78,8 +78,14 @@ def _GetDesiredFrontend(self, context:Context): # If we found something, ask the user if they want to use one. if len(foundFrontends) > 0: + # A lot of users seem to be confused by this frontend setup, so if there's only one interface, we will just use it. + if len(foundFrontends) == 1: + item = foundFrontends[0] + Logger.Info(f"Only one frontend was found [{str(item.Frontend)} - {str(item.Port)}] so we will use it for remote access.") + return (item.Port, str(item.Frontend)) + Logger.Blank() - Logger.Info("The following web interfaces were automatically discovered:") + Logger.Info("The following web interfaces were discovered:") count = 0 # List them in the order we found them, since we order the port list in by priority. for f in foundFrontends: @@ -87,7 +93,7 @@ def _GetDesiredFrontend(self, context:Context): Logger.Info(f" {count}) {str(f.Frontend).ljust(8)} - Port {str(f.Port)}") Logger.Blank() while True: - response = input("Enter the number next to the web interface you would like to use, or enter `m` to manually setup the web interface: ") + response = input("From the list above, enter the number of the web interface you would like to use for remote access; or enter `m` to manually setup the web interface: ") response = response.lower().strip() if response == "m": # Break to fall through to the manual setup. diff --git a/moonraker_installer/Installer.py b/moonraker_installer/Installer.py index 84d353b..1d97940 100644 --- a/moonraker_installer/Installer.py +++ b/moonraker_installer/Installer.py @@ -52,7 +52,7 @@ def _RunInternal(self): context = Context.LoadFromArgString(argObjectStr) # As soon as we have the user home make the log file. - Logger.InitFile(context.UserHomePath) + Logger.InitFile(context.UserHomePath, context.UserName) # Parse the original CmdLineArgs Logger.Debug("Parsing script cmd line args.") diff --git a/moonraker_installer/Logging.py b/moonraker_installer/Logging.py index 03de87d..5782c59 100644 --- a/moonraker_installer/Logging.py +++ b/moonraker_installer/Logging.py @@ -1,5 +1,7 @@ import os from datetime import datetime +# pylint: disable=import-error # Only exists on linux +import pwd # # Output Helpers @@ -20,11 +22,19 @@ class Logger: @staticmethod - def InitFile(userHomePath): + def InitFile(userHomePath:str, userName:str): try: Logger.OutputFilePath = os.path.join(userHomePath, "octoeverywhere-installer.log") + # pylint: disable=consider-using-with Logger.OutputFile = open(Logger.OutputFilePath, "w", encoding="utf-8") + + # Ensure the file is permission to the user who ran the script. + # Note we can't ref Util since it depends on the Logger. + uid = pwd.getpwnam(userName).pw_uid + gid = pwd.getpwnam(userName).pw_gid + # pylint: disable=no-member # Linux only + os.chown(Logger.OutputFilePath, uid, gid) except Exception as e: print("Failed to make log file. "+str(e)) diff --git a/moonraker_installer/Updater.py b/moonraker_installer/Updater.py index bd114f3..e00d26b 100644 --- a/moonraker_installer/Updater.py +++ b/moonraker_installer/Updater.py @@ -9,6 +9,7 @@ from .Configure import Configure from .Paths import Paths from .Service import Service +from .Util import Util # # This class is responsible for doing updates for all plugins and companions on this local system. @@ -124,6 +125,10 @@ def PlaceUpdateScriptInRoot(self, context:Context) -> bool: # Make sure to make it executable st = os.stat(updateFilePath) os.chmod(updateFilePath, st.st_mode | stat.S_IEXEC) + + # Ensure the user who launched the installer script has permissions to run it. + Util.SetFileOwnerRecursive(updateFilePath, context.UserName) + return True except Exception as e: Logger.Error("Failed to write updater script to user home. "+str(e)) From 624c449fabae615c2c6024d8279d89fe5507d18a Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Fri, 23 Feb 2024 20:10:05 -0800 Subject: [PATCH 027/328] Adding a fix for corupt config files when the installer is trying to read them. --- moonraker_installer/Frontend.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/moonraker_installer/Frontend.py b/moonraker_installer/Frontend.py index 6d938d7..1253021 100644 --- a/moonraker_installer/Frontend.py +++ b/moonraker_installer/Frontend.py @@ -279,17 +279,22 @@ def _TryToReadCurrentFrontendSetup(self, context:Context): if os.path.exists(filePath) is False: return (None, None) - config = configparser.ConfigParser() - config.read(filePath) - if config.has_section(Config.RelaySection) is False: - return (None, None) - if Config.RelayFrontEndPortKey not in config[Config.RelaySection]: + try: + config = configparser.ConfigParser() + config.read(filePath) + if config.has_section(Config.RelaySection) is False: + return (None, None) + if Config.RelayFrontEndPortKey not in config[Config.RelaySection]: + return (None, None) + portStr = config[Config.RelaySection][Config.RelayFrontEndPortKey] + frontendHint = None + if Config.RelayFrontEndTypeHintKey in config[Config.RelaySection]: + frontendHint = config[Config.RelaySection][Config.RelayFrontEndTypeHintKey] + return (portStr, frontendHint) + except Exception as e: + # There have been a few reports of this file being corrupt, so if it is, we will just fail and rewrite it. + Logger.Warn(f"Failed to read the frontend setup from the config file. {str(e)}") return (None, None) - portStr = config[Config.RelaySection][Config.RelayFrontEndPortKey] - frontendHint = None - if Config.RelayFrontEndTypeHintKey in config[Config.RelaySection]: - frontendHint = config[Config.RelaySection][Config.RelayFrontEndTypeHintKey] - return (portStr, frontendHint) # Writes the current frontend setup into the main OE service config From a9f10c8edc5b9e7005b335c0372f54d603f76634 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Tue, 27 Feb 2024 20:05:33 -0800 Subject: [PATCH 028/328] Refactoring the installer to make the system much better. --- .github/workflows/pylint.yml | 4 +- .pylintrc | 2 + .vscode/launch.json | 75 +++-- .vscode/settings.json | 14 +- bambu_octoeverywhere/__init__.py | 1 + bambu_octoeverywhere/__main__.py | 45 +++ bambu_octoeverywhere/bambuclient.py | 59 ++++ bambu_octoeverywhere/bambucommandhandler.py | 254 +++++++++++++++++ bambu_octoeverywhere/bambuhost.py | 232 ++++++++++++++++ bambu_octoeverywhere/bambuwebcamhelper.py | 42 +++ developer/runpylint.sh | 6 +- install.sh | 17 +- linux_host/__init__.py | 3 + .../config.py | 99 +++++-- .../logger.py | 4 +- .../secrets.py | 13 +- linux_host/startup.py | 89 ++++++ .../version.py | 0 moonraker_installer/DiscoveryObserver.py | 110 -------- moonraker_installer/ObserverConfigFile.py | 68 ----- moonraker_installer/__init__.py | 0 moonraker_octoeverywhere/__main__.py | 122 +++------ moonraker_octoeverywhere/moonrakerclient.py | 35 ++- .../moonrakercredentailmanager.py | 14 +- moonraker_octoeverywhere/moonrakerhost.py | 41 ++- .../moonrakerwebcamhelper.py | 3 +- .../observerconfigfile.py | 56 ---- octoeverywhere/commandhandler.py | 2 +- octoeverywhere/compat.py | 14 +- octoeverywhere/octopingpong.py | 3 +- py_installer/ConfigHelper.py | 174 ++++++++++++ py_installer/Configure.py | 140 ++++++++++ .../Context.py | 89 +++--- .../Discovery.py | 2 +- py_installer/DiscoveryCompanionAndBambu.py | 143 ++++++++++ .../Frontend.py | 85 +----- .../Installer.py | 41 +-- .../Linker.py | 4 +- .../Logging.py | 0 .../NetworkConnectors/BambuConnector.py | 259 ++++++++++++++++++ .../NetworkConnectors/MoonrakerConnector.py | 149 +--------- .../Paths.py | 0 .../Permissions.py | 24 +- .../ReadMe.py | 0 .../Service.py | 45 +-- .../TimeSync.py | 7 +- .../Uninstall.py | 31 +-- .../Updater.py | 16 +- {moonraker_installer => py_installer}/Util.py | 0 py_installer/__init__.py | 1 + .../__main__.py | 0 requirements.txt | 4 +- 52 files changed, 1890 insertions(+), 751 deletions(-) create mode 100644 bambu_octoeverywhere/__init__.py create mode 100644 bambu_octoeverywhere/__main__.py create mode 100644 bambu_octoeverywhere/bambuclient.py create mode 100644 bambu_octoeverywhere/bambucommandhandler.py create mode 100644 bambu_octoeverywhere/bambuhost.py create mode 100644 bambu_octoeverywhere/bambuwebcamhelper.py create mode 100644 linux_host/__init__.py rename {moonraker_octoeverywhere => linux_host}/config.py (75%) rename {moonraker_octoeverywhere => linux_host}/logger.py (92%) rename {moonraker_octoeverywhere => linux_host}/secrets.py (95%) create mode 100644 linux_host/startup.py rename {moonraker_octoeverywhere => linux_host}/version.py (100%) delete mode 100644 moonraker_installer/DiscoveryObserver.py delete mode 100644 moonraker_installer/ObserverConfigFile.py delete mode 100644 moonraker_installer/__init__.py delete mode 100644 moonraker_octoeverywhere/observerconfigfile.py create mode 100644 py_installer/ConfigHelper.py create mode 100644 py_installer/Configure.py rename {moonraker_installer => py_installer}/Context.py (77%) rename {moonraker_installer => py_installer}/Discovery.py (99%) create mode 100644 py_installer/DiscoveryCompanionAndBambu.py rename {moonraker_installer => py_installer}/Frontend.py (76%) rename {moonraker_installer => py_installer}/Installer.py (81%) rename {moonraker_installer => py_installer}/Linker.py (98%) rename {moonraker_installer => py_installer}/Logging.py (100%) create mode 100644 py_installer/NetworkConnectors/BambuConnector.py rename moonraker_installer/Configure.py => py_installer/NetworkConnectors/MoonrakerConnector.py (51%) rename {moonraker_installer => py_installer}/Paths.py (100%) rename {moonraker_installer => py_installer}/Permissions.py (80%) rename {moonraker_installer => py_installer}/ReadMe.py (100%) rename {moonraker_installer => py_installer}/Service.py (85%) rename {moonraker_installer => py_installer}/TimeSync.py (93%) rename {moonraker_installer => py_installer}/Uninstall.py (92%) rename {moonraker_installer => py_installer}/Updater.py (92%) rename {moonraker_installer => py_installer}/Util.py (100%) create mode 100644 py_installer/__init__.py rename {moonraker_installer => py_installer}/__main__.py (100%) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 2b34a50..c551a1b 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -26,4 +26,6 @@ jobs: pylint ./octoeverywhere/ pylint ./octoprint_octoeverywhere/ pylint ./moonraker_octoeverywhere/ - pylint ./moonraker_installer/ \ No newline at end of file + pylint ./bambu_octoeverywhere/ + pylint ./linux_host/ + pylint ./py_installer/ \ No newline at end of file diff --git a/.pylintrc b/.pylintrc index 67aa06d..30f41eb 100644 --- a/.pylintrc +++ b/.pylintrc @@ -22,6 +22,8 @@ disable= R1702, # Too many nested blocks R0801, # duplicate-code R0916, # Too many booleans in one if statment. + R1730, # consider-using-min-builtin + R1731, # consider-using-max-builtin # A comma-separated list of package or module names from where C extensions may diff --git a/.vscode/launch.json b/.vscode/launch.json index df28f3c..4b0a59c 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -5,36 +5,60 @@ "version": "0.2.0", "configurations": [ { - "name": "Moonraker Dev Module", - "type": "python", + "name": "Moonraker - PI - Single Instance", + "type": "debugpy", "request": "launch", "module": "moonraker_octoeverywhere", "justMyCode": false, "args": [ - // The module requires these aregs to be passed. These are examples of a typical default setup. - // This is obviously linux depdent, and is expected to be ran out of an installed repo with moonraker running. + // These args reflect the correct setup for a pi installed single instance of OctoEverywhere connecting to a local Moonraker. // The string is a urlBase64 encoded string of json. We base64 encode it to prevent any issues with command line args. // - // This is the single instance setup "eyJLbGlwcGVyQ29uZmlnRm9sZGVyIjogIi9ob21lL3BpL3ByaW50ZXJfZGF0YS9jb25maWciLCAiTW9vbnJha2VyQ29uZmlnRmlsZSI6ICIvaG9tZS9waS9wcmludGVyX2RhdGEvY29uZmlnL21vb25yYWtlci5jb25mIiwgIktsaXBwZXJMb2dGb2xkZXIiOiAiL2hvbWUvcGkvcHJpbnRlcl9kYXRhL2xvZ3MiLCAiTG9jYWxGaWxlU3RvcmFnZVBhdGgiOiAiL2hvbWUvcGkvcHJpbnRlcl9kYXRhL29jdG9ldmVyeXdoZXJlLXN0b3JlIiwgIlNlcnZpY2VOYW1lIjogIm9jdG9ldmVyeXdoZXJlIiwgIlZpcnR1YWxFbnZQYXRoIjogIi9ob21lL3BpL29jdG9ldmVyeXdoZXJlLWVudiIsICJSZXBvUm9vdEZvbGRlciI6ICIvaG9tZS9waS9vY3RvZXZlcnl3aGVyZSJ9", // - // This is the multi instance seutp - //"eyJLbGlwcGVyQ29uZmlnRm9sZGVyIjogIi9ob21lL3BpL3ByaW50ZXJfMV9kYXRhL2NvbmZpZyIsICJNb29ucmFrZXJDb25maWdGaWxlIjogIi9ob21lL3BpL3ByaW50ZXJfMV9kYXRhL2NvbmZpZy9tb29ucmFrZXIuY29uZiIsICJLbGlwcGVyTG9nRm9sZGVyIjogIi9ob21lL3BpL3ByaW50ZXJfMV9kYXRhL2xvZ3MiLCAiTG9jYWxGaWxlU3RvcmFnZVBhdGgiOiAiL2hvbWUvcGkvcHJpbnRlcl8xX2RhdGEvb2N0b2V2ZXJ5d2hlcmUtc3RvcmUiLCAiU2VydmljZU5hbWUiOiAib2N0b2V2ZXJ5d2hlcmUtMSIsICJWaXJ0dWFsRW52UGF0aCI6ICIvaG9tZS9waS9vY3RvZXZlcnl3aGVyZS1lbnYiLCAiUmVwb1Jvb3RGb2xkZXIiOiAiL2hvbWUvcGkvb2N0b2V2ZXJ5d2hlcmUifQ==", // We can optionally pass a dev config json object, which has dev specific overwrites we can make. "{\"LocalServerAddress\":\"\", \"LogLevel\":\"DEBUG\"}" ] }, { - "name": "Moonraker Dev Module - Observer Mode", - "type": "python", + "name": "Moonraker - PI - Multi Instance", + "type": "debugpy", "request": "launch", "module": "moonraker_octoeverywhere", "justMyCode": false, "args": [ - // The module requires these aregs to be passed. These are examples of a typical default setup. - // This is obviously linux depdent, and is expected to be ran out of an installed repo with moonraker running. + // These args reflect the correct setup for a pi installed multiple instances of OctoEverywhere connecting to a local Moonraker. These args target the first instance. // The string is a urlBase64 encoded string of json. We base64 encode it to prevent any issues with command line args. // + "eyJLbGlwcGVyQ29uZmlnRm9sZGVyIjogIi9ob21lL3BpL3ByaW50ZXJfMV9kYXRhL2NvbmZpZyIsICJNb29ucmFrZXJDb25maWdGaWxlIjogIi9ob21lL3BpL3ByaW50ZXJfMV9kYXRhL2NvbmZpZy9tb29ucmFrZXIuY29uZiIsICJLbGlwcGVyTG9nRm9sZGVyIjogIi9ob21lL3BpL3ByaW50ZXJfMV9kYXRhL2xvZ3MiLCAiTG9jYWxGaWxlU3RvcmFnZVBhdGgiOiAiL2hvbWUvcGkvcHJpbnRlcl8xX2RhdGEvb2N0b2V2ZXJ5d2hlcmUtc3RvcmUiLCAiU2VydmljZU5hbWUiOiAib2N0b2V2ZXJ5d2hlcmUtMSIsICJWaXJ0dWFsRW52UGF0aCI6ICIvaG9tZS9waS9vY3RvZXZlcnl3aGVyZS1lbnYiLCAiUmVwb1Jvb3RGb2xkZXIiOiAiL2hvbWUvcGkvb2N0b2V2ZXJ5d2hlcmUifQ==", + // We can optionally pass a dev config json object, which has dev specific overwrites we can make. + "{\"LocalServerAddress\":\"\", \"LogLevel\":\"DEBUG\"}" + ] + }, + { + "name": "Bambu Connect - PI - Instnace 1", + "type": "debugpy", + "request": "launch", + "module": "bambu_octoeverywhere", + "justMyCode": false, + "args": [ + // These args reflect the correct setup for a pi installed with the Bambu Connect version of the plugin. These args target the first instance. + // + // { "ServiceName": "octoeverywhere-bambu", "VirtualEnvPath":"/home/pi/octoeverywhere-env", "RepoRootFolder":"/home/pi/octoeverywhere/", "LogFolder":"/home/pi/.octoeverywhere-bambu/logs", "LocalFileStoragePath":"/home/pi/.octoeverywhere-bambu/octoeverywhere-store", "ConfigFolder":"/home/pi/.octoeverywhere-bambu/", "InstanceStr":"1" } + "eyAiU2VydmljZU5hbWUiOiAib2N0b2V2ZXJ5d2hlcmUtYmFtYnUiLCAiVmlydHVhbEVudlBhdGgiOiIvaG9tZS9waS9vY3RvZXZlcnl3aGVyZS1lbnYiLCAiUmVwb1Jvb3RGb2xkZXIiOiIvaG9tZS9waS9vY3RvZXZlcnl3aGVyZS8iLCAiTG9nRm9sZGVyIjoiL2hvbWUvcGkvLm9jdG9ldmVyeXdoZXJlLWJhbWJ1L2xvZ3MiLCAiTG9jYWxGaWxlU3RvcmFnZVBhdGgiOiIvaG9tZS9waS8ub2N0b2V2ZXJ5d2hlcmUtYmFtYnUvb2N0b2V2ZXJ5d2hlcmUtc3RvcmUiLCAiQ29uZmlnRm9sZGVyIjoiL2hvbWUvcGkvLm9jdG9ldmVyeXdoZXJlLWJhbWJ1LyIsICJJbnN0YW5jZVN0ciI6IjEiIH0=", + // We can optionally pass a dev config json object, which has dev specific overwrites we can make. + "{\"LocalServerAddress\":\"\", \"LogLevel\":\"DEBUG\"}" + ] + }, + { + "name": "Companion - PI - Instnace 1", + "type": "debugpy", + "request": "launch", + "module": "moonraker_octoeverywhere", + "justMyCode": false, + "args": [ + // These args reflect the correct setup for a pi installed with the Companion version of the plugin. These args target the first instance. + // // This is the single instance setup "eyJLbGlwcGVyQ29uZmlnRm9sZGVyIjogIi9ob21lL3BpLy5vY3RvZXZlcnl3aGVyZS1vYnNlcnZlci0xL2NvbmZpZyIsICJNb29ucmFrZXJDb25maWdGaWxlIjogbnVsbCwgIktsaXBwZXJMb2dGb2xkZXIiOiAiL2hvbWUvcGkvLm9jdG9ldmVyeXdoZXJlLW9ic2VydmVyLTEvbG9ncyIsICJMb2NhbEZpbGVTdG9yYWdlUGF0aCI6ICIvaG9tZS9waS8ub2N0b2V2ZXJ5d2hlcmUtb2JzZXJ2ZXItMS9vY3RvZXZlcnl3aGVyZS1zdG9yZSIsICJTZXJ2aWNlTmFtZSI6ICJvY3RvZXZlcnl3aGVyZS1vYnNlcnZlcjEiLCAiVmlydHVhbEVudlBhdGgiOiAiL2hvbWUvcGkvb2N0b2V2ZXJ5d2hlcmUtZW52IiwgIlJlcG9Sb290Rm9sZGVyIjogIi9ob21lL3BpL29jdG9ldmVyeXdoZXJlIiwgIklzT2JzZXJ2ZXIiOiB0cnVlLCAiT2JzZXJ2ZXJDb25maWdGaWxlUGF0aCI6ICIvaG9tZS9waS8ub2N0b2V2ZXJ5d2hlcmUtb2JzZXJ2ZXItMS9jb25maWcvb2N0b2V2ZXJ5d2hlcmUtb2JzZXJ2ZXIuY2ZnIiwgIk9ic2VydmVySW5zdGFuY2VJZFN0ciI6ICIxIn0=", // @@ -45,10 +69,10 @@ ] }, { - "name": "Moonraker Installer Module", - "type": "python", + "name": "Installer - Local Moonraker", + "type": "debugpy", "request": "launch", - "module": "moonraker_installer", + "module": "py_installer", "justMyCode": false, "args": [ // The module requires this json object to be passed. @@ -58,21 +82,34 @@ ] }, { - "name": "Moonraker Installer Module - Observer", - "type": "python", + "name": "Installer - Companion", + "type": "debugpy", + "request": "launch", + "module": "py_installer", + "justMyCode": false, + "args": [ + // The module requires this json object to be passed. + // Normally the install.sh script runs, ensure everything is installed, creates a virtural env, and then runs this modlue giving it these args. + // But for debugging, we can skip that assuming it's already been ran. + "{\"OE_REPO_DIR\":\"/home/pi/octoeverywhere\",\"OE_ENV\":\"/home/pi/octoeverywhere-env\",\"USERNAME\":\"pi\",\"USER_HOME\":\"/home/pi\",\"CMD_LINE_ARGS\":\"-debug -skipsudoactions -companion\"}" + ] + }, + { + "name": "Installer - Bambu Connect", + "type": "debugpy", "request": "launch", - "module": "moonraker_installer", + "module": "py_installer", "justMyCode": false, "args": [ // The module requires this json object to be passed. // Normally the install.sh script runs, ensure everything is installed, creates a virtural env, and then runs this modlue giving it these args. // But for debugging, we can skip that assuming it's already been ran. - "{\"OE_REPO_DIR\":\"/home/pi/octoeverywhere\",\"OE_ENV\":\"/home/pi/octoeverywhere-env\",\"USERNAME\":\"pi\",\"USER_HOME\":\"/home/pi\",\"CMD_LINE_ARGS\":\"-debug -skipsudoactions -observer\"}" + "{\"OE_REPO_DIR\":\"/home/pi/octoeverywhere\",\"OE_ENV\":\"/home/pi/octoeverywhere-env\",\"USERNAME\":\"pi\",\"USER_HOME\":\"/home/pi\",\"CMD_LINE_ARGS\":\"-debug -skipsudoactions -bambu\"}" ] }, { "name": "OctoPrint Dev Module", - "type": "python", + "type": "debugpy", "request": "launch", "module": "octoprint_octoeverywhere", "justMyCode": true diff --git a/.vscode/settings.json b/.vscode/settings.json index 48fd2b2..92773f3 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -9,6 +9,12 @@ "asvc", "authed", "backoff", + "Bambu", + "bambuclient", + "bambucommandhandler", + "bambuhost", + "bambuwebcamhelper", + "bblp", "bootstraper", "boundarydonotcross", "brotli", @@ -19,6 +25,7 @@ "commandhandler", "Commonize", "comms", + "companionconfigfile", "continuousprint", "Creality", "Creality's", @@ -33,6 +40,7 @@ "DGRAM", "didnt", "dnspython", + "esac", "filamentchange", "filemetadatacache", "finalsnap", @@ -87,13 +95,13 @@ "moonrakerdatabase", "moonrakerhost", "moonrakerwebcamhelper", + "mqtt", "msgcount", "multicam", "myprinter", "nbsp", "noatuoselect", "notificationshandler", - "observerconfigfile", "Ocoto", "ocotomessage", "Octo", @@ -124,6 +132,7 @@ "oprint", "ostype", "ostypeidentifier", + "paho", "peasy", "permissioned", "PKGLIST", @@ -137,6 +146,7 @@ "Proto", "proxying", "Pursa", + "pushall", "pushd", "Pylint", "pythoncompat", @@ -148,6 +158,7 @@ "referer", "releaseinfo", "repeattimer", + "reqs", "requestsutils", "routable", "sdcard", @@ -181,6 +192,7 @@ "unauthed", "updatemanager", "urandom", + "userdata", "userinteractionneeded", "venv", "VIEWMODELS", diff --git a/bambu_octoeverywhere/__init__.py b/bambu_octoeverywhere/__init__.py new file mode 100644 index 0000000..564091d --- /dev/null +++ b/bambu_octoeverywhere/__init__.py @@ -0,0 +1 @@ +# Need to make this a module diff --git a/bambu_octoeverywhere/__main__.py b/bambu_octoeverywhere/__main__.py new file mode 100644 index 0000000..5ea9c23 --- /dev/null +++ b/bambu_octoeverywhere/__main__.py @@ -0,0 +1,45 @@ +import sys + +from linux_host.startup import Startup +from linux_host.startup import ConfigDataTypes + +from .bambuhost import BambuHost + +if __name__ == '__main__': + + # This is a helper class, to keep the startup logic common. + s = Startup() + + # Try to parse the config + jsonConfigStr = None + try: + # Get the json from the process args. + jsonConfig = s.GetJsonFromArgs(sys.argv) + + # + # 1) Parse the common, required args. + # + ServiceName = s.GetConfigVarAndValidate(jsonConfig, "ServiceName", ConfigDataTypes.String) + VirtualEnvPath = s.GetConfigVarAndValidate(jsonConfig, "VirtualEnvPath", ConfigDataTypes.Path) + RepoRootFolder = s.GetConfigVarAndValidate(jsonConfig, "RepoRootFolder", ConfigDataTypes.Path) + LocalFileStoragePath = s.GetConfigVarAndValidate(jsonConfig, "LocalFileStoragePath", ConfigDataTypes.Path) + LogFolder = s.GetConfigVarAndValidate(jsonConfig, "LogFolder", ConfigDataTypes.Path) + ConfigFolder = s.GetConfigVarAndValidate(jsonConfig, "ConfigFolder", ConfigDataTypes.Path) + InstanceStr = s.GetConfigVarAndValidate(jsonConfig, "InstanceStr", ConfigDataTypes.String) + + except Exception as e: + s.PrintErrorAndExit(f"Exception while loading json config. Error:{str(e)}, Config: {jsonConfigStr}") + + # For debugging, we also allow an optional dev object to be passed. + devConfig_CanBeNone = s.GetDevConfigIfAvailable(sys.argv) + + # Run! + try: + # Create and run the main host! + host = BambuHost(ConfigFolder, LogFolder, devConfig_CanBeNone) + host.RunBlocking(ConfigFolder, LocalFileStoragePath, RepoRootFolder, devConfig_CanBeNone) + except Exception as e: + s.PrintErrorAndExit(f"Exception leaked from main bambu host class. Error:{str(e)}") + + # If we exit here, it's due to an error, since RunBlocking should be blocked forever. + sys.exit(1) diff --git a/bambu_octoeverywhere/bambuclient.py b/bambu_octoeverywhere/bambuclient.py new file mode 100644 index 0000000..1f1245f --- /dev/null +++ b/bambu_octoeverywhere/bambuclient.py @@ -0,0 +1,59 @@ +import logging +import ssl +import json +import threading + +import paho.mqtt.client as mqtt + +from linux_host.config import Config + +class BambuClient: + + def __init__(self, logger:logging.Logger, config:Config) -> None: + self.Logger = logger + + self.IpOrHostname = config.GetStr(Config.SectionCompanion, Config.CompanionKeyIpOrHostname, None) + self.PortStr = config.GetStr(Config.SectionCompanion, Config.CompanionKeyPort, None) + self.AccessToken = config.GetStr(Config.SectionBambu, Config.BambuAccessToken, None) + self.PrinterSn = config.GetStr(Config.SectionBambu, Config.BambuPrinterSn, None) + if self.IpOrHostname is None or self.PortStr is None or self.AccessToken is None or self.PrinterSn is None: + raise Exception("Missing required args from the config") + + self.client = mqtt.Client(mqtt.CallbackAPIVersion.VERSION2) + self.client.tls_set(tls_version=ssl.PROTOCOL_TLS, cert_reqs=ssl.CERT_NONE) + self.client.tls_insecure_set(True) + self.client.username_pw_set("bblp", self.AccessToken) + self.client.on_connect = self.OnConnect + self.client.on_message = self.OnMessage + self.client.on_disconnect = self.OnDisconnect + self.client.connect(self.IpOrHostname, int(self.PortStr), 60) + t = threading.Thread(target=self.Worker) + t.start() + + + def DoesNothing(self): + pass + + + def Worker(self): + self.client.loop_forever() + + + def OnConnect(self, client, userdata, flags, reason_code, properties): + self.Logger.warn("MQTT connected") + client.subscribe(f"device/{self.PrinterSn}/report") + + + def OnDisconnect(self, client, userdata, disconnect_flags, reason_code, properties): + self.Logger.warn("MQTT disconnected") + + + def OnMessage(self, client, userdata, msg): + try: + doc = json.loads(msg.payload) + self.Logger.info("Bambu "+json.dumps(doc, indent=3)) + if doc is None: + return + self.client.publish(f"device/{self.PrinterSn}/request", '{{"pushing": {{ "sequence_id": 1, "command": "pushall"}}, "user_id":"1234567890"}}') + except Exception: + pass diff --git a/bambu_octoeverywhere/bambucommandhandler.py b/bambu_octoeverywhere/bambucommandhandler.py new file mode 100644 index 0000000..4c89245 --- /dev/null +++ b/bambu_octoeverywhere/bambucommandhandler.py @@ -0,0 +1,254 @@ +#import json + +from octoeverywhere.commandhandler import CommandHandler, CommandResponse + +# This class implements the Platform Command Handler Interface +class BambuCommandHandler: + + def __init__(self, logger) -> None: + self.Logger = logger + + + # !! Platform Command Handler Interface Function !! + # + # This must return the common "JobStatus" dict or None on failure. + # The format of this must stay consistent with OctoPrint and the service. + # Returning None send back the NoHostConnected error, assuming that the plugin isn't connected to the host or the host isn't + # connected to the printer's firmware. + # + # See the JobStatusV2 class in the service for the object definition. + # + def GetCurrentJobStatus(self): + # result = MoonrakerClient.Get().SendJsonRpcRequest("printer.objects.query", + # { + # "objects": { + # "print_stats": None, # Needed for many things, including GetPrintTimeRemainingEstimateInSeconds_WithPrintStatsAndVirtualSdCardResult + # "gcode_move": None, # Needed for GetPrintTimeRemainingEstimateInSeconds_WithPrintStatsAndVirtualSdCardResult to get the current speed + # "virtual_sdcard": None, # Needed for many things, including GetPrintTimeRemainingEstimateInSeconds_WithPrintStatsAndVirtualSdCardResult + # "extruder": None, # Needed for temps + # "heater_bed": None, # Needed for temps + # # "webhooks": None, + # # "extruder": None, + # # "bed_mesh": None, + # } + # }) + # # Validate + # if result.HasError(): + # self.Logger.error("MoonrakerCommandHandler failed GetCurrentJobStatus() query. "+result.GetLoggingErrorStr()) + # return None + + # # Get the result. + # res = result.GetResult() + + # # Map the state + # state = "idle" + # if "status" in res and "print_stats" in res["status"] and "state" in res["status"]["print_stats"]: + # # https://moonraker.readthedocs.io/en/latest/printer_objects/#print_stats + # mrState = res["status"]["print_stats"]["state"] + # if mrState == "standby": + # state = "idle" + # elif mrState == "printing": + # # This is a special case, we consider "warmingup" a subset of printing. + # if MoonrakerClient.Get().GetMoonrakerCompat().CheckIfPrinterIsWarmingUp_WithPrintStats(result): + # state = "warmingup" + # else: + # state = "printing" + # elif mrState == "paused": + # state = "paused" + # elif mrState == "complete": + # state = "complete" + # elif mrState == "cancelled": + # state = "cancelled" + # elif mrState == "error": + # state = "error" + # else: + # self.Logger.warn("Unknown mrState returned from print_stats: "+str(mrState)) + # else: + # self.Logger.warn("MoonrakerCommandHandler failed to find the print_stats.status") + + # # TODO - If in an error state, set some context as to why. + # errorStr_CanBeNone = None + + # # Get current layer info + # # None = The platform doesn't provide it. + # # 0 = The platform provider it, but there's no info yet. + # # # = The values + # # Note this is similar to how we also do it for notifications. + # currentLayerInt = None + # totalLayersInt = None + # currentLayerRaw, totalLayersRaw = MoonrakerClient.Get().GetMoonrakerCompat().GetCurrentLayerInfo() + # if totalLayersRaw is not None and totalLayersRaw > 0 and currentLayerRaw is not None and currentLayerRaw >= 0: + # currentLayerInt = int(currentLayerRaw) + # totalLayersInt = int(totalLayersRaw) + + # # Get duration and filename. + # durationSec = 0 + # fileName = "" + # if "status" in res and "print_stats" in res["status"]: + # ps = res["status"]["print_stats"] + # # We choose to use print_duration over "total_duration" so we only show the time actually spent printing. This is consistent across platforms. + # if "print_duration" in ps: + # durationSec = int(ps["print_duration"]) + # if "filename" in ps: + # fileName = ps["filename"] + + # # If we have a file name, try to get the current filament usage. + # filamentUsageMm = 0 + # if fileName is not None and len(fileName) > 0: + # filamentUsageMm = FileMetadataCache.Get().GetEstimatedFilamentUsageMm(fileName) + + # # Get the progress + # progress = 0.0 + # if "status" in res and "virtual_sdcard" in res["status"]: + # vs = res["status"]["virtual_sdcard"] + # if "progress" in vs: + # # Convert progress 0->1 to 0->100 + # progress = vs["progress"] * 100.0 + + # # Time left can be hard to compute correctly, so use the common function to do it based + # # on what we can get as a best effort. + # timeLeftSec = MoonrakerClient.Get().GetMoonrakerCompat().GetPrintTimeRemainingEstimateInSeconds_WithPrintStatsVirtualSdCardAndGcodeMoveResult(result) + + # # Get the current temps if possible. + # hotendActual = 0.0 + # hotendTarget = 0.0 + # bedTarget = 0.0 + # bedActual = 0.0 + # if "status" in res and "extruder" in res["status"]: + # extruder = res["status"]["extruder"] + # if "temperature" in extruder: + # hotendActual = round(float(extruder["temperature"]), 2) + # if "target" in extruder: + # hotendTarget = round(float(extruder["target"]), 2) + # if "status" in res and "heater_bed" in res["status"]: + # heater_bed = res["status"]["heater_bed"] + # if "temperature" in heater_bed: + # bedActual = round(float(heater_bed["temperature"]), 2) + # if "target" in heater_bed: + # bedTarget = round(float(heater_bed["target"]), 2) + + # Build the object and return. + return { + "State": "error", + "Error": "bambu", + } + + # return { + # "State": state, + # "Error": errorStr_CanBeNone, + # "CurrentPrint": + # { + # "Progress" : progress, + # "DurationSec" : durationSec, + # "TimeLeftSec" : timeLeftSec, + # "FileName" : fileName, + # "EstTotalFilUsedMm" : filamentUsageMm, + # "CurrentLayer": currentLayerInt, + # "TotalLayers": totalLayersInt, + # "Temps": { + # "BedActual": bedActual, + # "BedTarget": bedTarget, + # "HotendActual": hotendActual, + # "HotendTarget": hotendTarget, + # } + # } + # } + + + # !! Platform Command Handler Interface Function !! + # This must return the platform version as a string. + def GetPlatformVersionStr(self): + # We don't supply this for moonraker at the moment. + return "1.0.0" + + + # !! Platform Command Handler Interface Function !! + # This must check that the printer state is valid for the pause and the plugin is connected to the host. + # If not, it must return the correct two error codes accordingly. + # This must return a CommandResponse. + def ExecutePause(self, smartPause, suppressNotificationBool, disableHotendBool, disableBedBool, zLiftMm, retractFilamentMm, showSmartPausePopup) -> CommandResponse: + # Check the state and that we have a connection to the host. + result = self._CheckIfConnectedAndForExpectedStates(["printing"]) + if result is not None: + return result + + # The smart pause logic handles all pause commands. + #return SmartPause.Get().ExecuteSmartPause(suppressNotificationBool) + return CommandResponse.Success(None) + + + # !! Platform Command Handler Interface Function !! + # This must check that the printer state is valid for the resume and the plugin is connected to the host. + # If not, it must return the correct two error codes accordingly. + # This must return a CommandResponse. + def ExecuteResume(self) -> CommandResponse: + # Check the state and that we have a connection to the host. + result = self._CheckIfConnectedAndForExpectedStates(["paused"]) + if result is not None: + return result + + # # Do the resume. + # result = MoonrakerClient.Get().SendJsonRpcRequest("printer.print.resume", {}) + # if result.HasError(): + # self.Logger.error("ExecuteResume failed to request resume. "+result.GetLoggingErrorStr()) + # return CommandResponse.Error(400, "Failed to request resume") + + # # Check the response + # if result.GetResult() != "ok": + # self.Logger.error("ExecuteResume got an invalid request response. "+json.dumps(result.GetResult())) + # return CommandResponse.Error(400, "Invalid request response.") + + return CommandResponse.Success(None) + + + # !! Platform Command Handler Interface Function !! + # This must check that the printer state is valid for the cancel and the plugin is connected to the host. + # If not, it must return the correct two error codes accordingly. + # This must return a CommandResponse. + def ExecuteCancel(self) -> CommandResponse: + # Check the state and that we have a connection to the host. + result = self._CheckIfConnectedAndForExpectedStates(["printing","paused"]) + if result is not None: + return result + + # Do the resume. + # result = MoonrakerClient.Get().SendJsonRpcRequest("printer.print.cancel", {}) + # if result.HasError(): + # self.Logger.error("ExecuteCancel failed to request cancel. "+result.GetLoggingErrorStr()) + # return CommandResponse.Error(400, "Failed to request cancel") + + # # Check the response + # if result.GetResult() != "ok": + # self.Logger.error("ExecuteCancel got an invalid request response. "+json.dumps(result.GetResult())) + # return CommandResponse.Error(400, "Invalid request response.") + + return CommandResponse.Success(None) + + + # Checks if the printer is connected and in the correct state (or states) + # If everything checks out, returns None. Otherwise it returns a CommandResponse + def _CheckIfConnectedAndForExpectedStates(self, stateArray) -> CommandResponse: + # Only allow the pause if the print state is printing, otherwise the system seems to get confused. + # result = MoonrakerClient.Get().SendJsonRpcRequest("printer.objects.query", + # { + # "objects": { + # "print_stats": None + # } + # }) + # if result.HasError(): + # if result.ErrorCode == JsonRpcResponse.OE_ERROR_WS_NOT_CONNECTED: + # self.Logger.error("Command failed because the printer is no connected. "+result.GetLoggingErrorStr()) + # return CommandResponse.Error(CommandHandler.c_CommandError_HostNotConnected, "Printer Not Connected") + # self.Logger.error("Command failed to get state. "+result.GetLoggingErrorStr()) + # return CommandResponse.Error(500, "Error Getting State") + # res = result.GetResult() + # if "status" not in res or "print_stats" not in res["status"] or "state" not in res["status"]["print_stats"]: + # self.Logger.error("Command failed to get state, state not found in dict.") + # return CommandResponse.Error(500, "Error Getting State From Dict") + # state = res["status"]["print_stats"]["state"] + # for s in stateArray: + # if s == state: + # return None + + # self.Logger.warn("Command failed, printer "+state+" not the expected states.") + return CommandResponse.Error(CommandHandler.c_CommandError_InvalidPrinterState, "Wrong State") diff --git a/bambu_octoeverywhere/bambuhost.py b/bambu_octoeverywhere/bambuhost.py new file mode 100644 index 0000000..3642d3d --- /dev/null +++ b/bambu_octoeverywhere/bambuhost.py @@ -0,0 +1,232 @@ +import logging +import traceback + +from octoeverywhere.mdns import MDns +from octoeverywhere.sentry import Sentry +from octoeverywhere.hostcommon import HostCommon +from octoeverywhere.telemetry import Telemetry +from octoeverywhere.webcamhelper import WebcamHelper +from octoeverywhere.octopingpong import OctoPingPong +from octoeverywhere.commandhandler import CommandHandler +from octoeverywhere.octoeverywhereimpl import OctoEverywhere +from octoeverywhere.Proto.ServerHost import ServerHost +from octoeverywhere.compat import Compat + +from linux_host.config import Config +from linux_host.secrets import Secrets +from linux_host.version import Version +from linux_host.logger import LoggerInit + +from .bambuclient import BambuClient +from .bambucommandhandler import BambuCommandHandler +from .bambuwebcamhelper import BambuWebcamHelper + +# This file is the main host for the bambu service. +class BambuHost: + + def __init__(self, configDir:str, logDir:str, devConfig_CanBeNone) -> None: + # When we create our class, make sure all of our core requirements are created. + self.Secrets = None + + # Let the compat system know this is an Bambu host. + Compat.SetIsBambu(True) + + try: + # First, we need to load our config. + # Note that the config MUST BE WRITTEN into this folder, that's where the setup installer is going to look for it. + # If this fails, it will throw. + self.Config = Config(configDir) + + # Setup the logger. + logLevelOverride_CanBeNone = self.GetDevConfigStr(devConfig_CanBeNone, "LogLevel") + self.Logger = LoggerInit.GetLogger(self.Config, logDir, logLevelOverride_CanBeNone) + self.Config.SetLogger(self.Logger) + + # Init sentry, since it's needed for Exceptions. + Sentry.Init(self.Logger, "bambu", True) + + except Exception as e: + tb = traceback.format_exc() + print("Failed to init Bambu Host! "+str(e) + "; "+str(tb)) + # Raise the exception so we don't continue. + raise + + + def RunBlocking(self, configPath, localStorageDir, repoRoot, devConfig_CanBeNone): + # Do all of this in a try catch, so we can log any issues before exiting + try: + self.Logger.info("##################################") + self.Logger.info("#### OctoEverywhere Starting #####") + self.Logger.info("##################################") + + # Find the version of the plugin, this is required and it will throw if it fails. + pluginVersionStr = Version.GetPluginVersion(repoRoot) + self.Logger.info("Plugin Version: %s", pluginVersionStr) + + # Before the first time setup, we must also init the Secrets class and do the migration for the printer id and private key, if needed. + # As of 8/15/2023, we don't store any sensitive things in teh config file, since all config files are sometimes backed up publicly. + self.Secrets = Secrets(self.Logger, localStorageDir) + + # Now, detect if this is a new instance and we need to init our global vars. If so, the setup script will be waiting on this. + self.DoFirstTimeSetupIfNeeded() + + # Get our required vars + printerId = self.GetPrinterId() + privateKey = self.GetPrivateKey() + + # Unpack any dev vars that might exist + DevLocalServerAddress_CanBeNone = self.GetDevConfigStr(devConfig_CanBeNone, "LocalServerAddress") + if DevLocalServerAddress_CanBeNone is not None: + self.Logger.warning("~~~ Using Local Dev Server Address: %s ~~~", DevLocalServerAddress_CanBeNone) + + # Init Sentry, but it won't report since we are in dev mode. + Telemetry.Init(self.Logger) + if DevLocalServerAddress_CanBeNone is not None: + Telemetry.SetServerProtocolAndDomain("http://"+DevLocalServerAddress_CanBeNone) + + # Init the mdns client + MDns.Init(self.Logger, localStorageDir) + + + + # # Setup the http requester. We default to port 80 and assume the frontend can be found there. + # # TODO - parse nginx to see what front ends exist and make them switchable + # # TODO - detect HTTPS port if 80 is not bound. + # frontendPort = self.Config.GetInt(Config.RelaySection, Config.RelayFrontEndPortKey, 80) + # self.Logger.info("Setting up relay with frontend port %s", str(frontendPort)) + # OctoHttpRequest.SetLocalHttpProxyPort(frontendPort) + # OctoHttpRequest.SetLocalHttpProxyIsHttps(False) + # OctoHttpRequest.SetLocalOctoPrintPort(frontendPort) + + # Init the ping pong helper. + OctoPingPong.Init(self.Logger, localStorageDir, printerId) + if DevLocalServerAddress_CanBeNone is not None: + OctoPingPong.Get().DisablePrimaryOverride() + + # Setup the snapshot helper + # TODO + webcamHelper = BambuWebcamHelper(self.Logger) + WebcamHelper.Init(self.Logger, webcamHelper, localStorageDir) + + # Setup the command handler + # TODO - Notification handler + CommandHandler.Init(self.Logger, None, BambuCommandHandler(self.Logger)) + + c = BambuClient(self.Logger, self.Config) + c.DoesNothing() + + # Now start the main runner! + OctoEverywhereWsUri = HostCommon.c_OctoEverywhereOctoClientWsUri + if DevLocalServerAddress_CanBeNone is not None: + OctoEverywhereWsUri = "ws://"+DevLocalServerAddress_CanBeNone+"/octoclientws" + # TODO update types + oe = OctoEverywhere(OctoEverywhereWsUri, printerId, privateKey, self.Logger, self, self, pluginVersionStr, ServerHost.Moonraker, False) + oe.RunBlocking() + except Exception as e: + Sentry.Exception("!! Exception thrown out of main host run function.", e) + + # Allow the loggers to flush before we exit + try: + self.Logger.info("##################################") + self.Logger.info("#### OctoEverywhere Exiting ######") + self.Logger.info("##################################") + logging.shutdown() + except Exception as e: + print("Exception in logging.shutdown "+str(e)) + + + # Ensures all required values are setup and valid before starting. + def DoFirstTimeSetupIfNeeded(self): + # Try to get the printer id from the config. + printerId = self.GetPrinterId() + if HostCommon.IsPrinterIdValid(printerId) is False: + if printerId is None: + self.Logger.info("No printer id was found, generating one now!") + else: + self.Logger.info("An invalid printer id was found [%s], regenerating!", str(printerId)) + + # Make a new, valid, key + printerId = HostCommon.GeneratePrinterId() + + # Save it + self.Secrets.SetPrinterId(printerId) + self.Logger.info("New printer id created: %s", printerId) + + privateKey = self.GetPrivateKey() + if HostCommon.IsPrivateKeyValid(privateKey) is False: + if privateKey is None: + self.Logger.info("No private key was found, generating one now!") + else: + self.Logger.info("An invalid private key was found [%s], regenerating!", str(privateKey)) + + # Make a new, valid, key + privateKey = HostCommon.GeneratePrivateKey() + + # Save it + self.Secrets.SetPrivateKey(privateKey) + self.Logger.info("New private key created.") + + + # Returns None if no printer id has been set. + def GetPrinterId(self): + return self.Secrets.GetPrinterId() + + + # Returns None if no private id has been set. + def GetPrivateKey(self): + return self.Secrets.GetPrivateKey() + + + # Tries to load a dev config option as a string. + # If not found or it fails, this return None + def GetDevConfigStr(self, devConfig, value): + if devConfig is None: + return None + if value in devConfig: + v = devConfig[value] + if v is not None and len(v) > 0 and v != "None": + return v + return None + + + # UiPopupInvoker Interface function - Sends a UI popup message for various uses. + # Must stay in sync with the OctoPrint handler! + # title - string, the title text. + # text - string, the message. + # type - string, [notice, info, success, error] the type of message shown. + # actionText - string, if not None or empty, this is the text to show on the action button or text link. + # actionLink - string, if not None or empty, this is the URL to show on the action button or text link. + # onlyShowIfLoadedViaOeBool - bool, if set, the message should only be shown on browsers loading the portal from OE. + def ShowUiPopup(self, title:str, text:str, msgType:str, actionText:str, actionLink:str, showForSec:int, onlyShowIfLoadedViaOeBool:bool): + # This isn't supported on Bambu + pass + + + # + # StatusChangeHandler Interface - Called by the OctoEverywhere logic when the server connection has been established. + # + def OnPrimaryConnectionEstablished(self, octoKey, connectedAccounts): + self.Logger.info("Primary Connection To OctoEverywhere Established - We Are Ready To Go!") + + # Check if this printer is unlinked, if so add a message to the log to help the user setup the printer if desired. + # This would be if the skipped the printer link or missed it in the setup script. + if connectedAccounts is None or len(connectedAccounts) == 0: + self.Logger.warning("") + self.Logger.warning("") + self.Logger.warning("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + self.Logger.warning("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + self.Logger.warning(" This Plugin Isn't Connected To OctoEverywhere! ") + self.Logger.warning(" Use the following link to finish the setup and get remote access:") + self.Logger.warning(" %s", HostCommon.GetAddPrinterUrl(self.GetPrinterId(), False)) + self.Logger.warning("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + self.Logger.warning("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + self.Logger.warning("") + self.Logger.warning("") + + + # + # StatusChangeHandler Interface - Called by the OctoEverywhere logic when a plugin update is required for this client. + # + def OnPluginUpdateRequired(self): + self.Logger.error("!!! A Plugin Update Is Required -- If This Plugin Isn't Updated It Might Stop Working !!!") + self.Logger.error("!!! Please use the update manager in Mainsail of Fluidd to update this plugin !!!") diff --git a/bambu_octoeverywhere/bambuwebcamhelper.py b/bambu_octoeverywhere/bambuwebcamhelper.py new file mode 100644 index 0000000..456db44 --- /dev/null +++ b/bambu_octoeverywhere/bambuwebcamhelper.py @@ -0,0 +1,42 @@ +import logging + + +#from octoeverywhere.sentry import Sentry +from octoeverywhere.webcamhelper import WebcamSettingItem#, WebcamHelper + +# This class implements the webcam platform helper interface for bambu. +class BambuWebcamHelper(): + + def __init__(self, logger:logging.Logger) -> None: + self.Logger = logger + + + # !! Interface Function !! + # This must return an array of WebcamSettingItems. + # Index 0 is used as the default webcam. + # The order the webcams are returned is the order the user will see in any selection UIs. + # Returns None on failure. + def GetWebcamConfig(self): + return [WebcamSettingItem("Default", "self.SnapshotUrl", "self.StreamUrl", "self.FlipH", "self.FlipV", "self.Rotation")] + + # # Kick the settings worker since the webcam was accessed. + # self.KickOffWebcamSettingsUpdate() + + # # Grab the lock to see what we should be returning. + # with self.ResultsLock: + # # If auto settings are enabled, return any cached auto settings that we found. + # # If we have anything, make a copy of the array and return it. + # if self.EnableAutoSettings: + # if len(self.AutoSettingsResults) != 0: + # results = [] + # for i in self.AutoSettingsResults: + # results.append(i) + # return results + # # If we don't have auto settings enabled or we don't have any results, return what we have in memory. + # # This will either be the default values or a values that the user has set. + # item = WebcamSettingItem("Default", self.SnapshotUrl, self.StreamUrl, self.FlipH, self.FlipV, self.Rotation) + # # Validate the settings, but always return them. + # item.Validate(self.Logger) + # return [ + # item + # ] diff --git a/developer/runpylint.sh b/developer/runpylint.sh index 2e5ffbf..1e96df7 100755 --- a/developer/runpylint.sh +++ b/developer/runpylint.sh @@ -5,5 +5,9 @@ echo "Testing OctoPrint Module..." pylint ./octoprint_octoeverywhere/ echo "Testing Moonraker Module..." pylint ./moonraker_octoeverywhere/ +echo "Testing Linux Host Module..." +pylint ./linux_host/ +echo "Testing Bambu Module..." +pylint ./bambu_octoeverywhere/ echo "Testing Moonraker Installer Module..." -pylint ./moonraker_installer/ \ No newline at end of file +pylint ./py_installer/ \ No newline at end of file diff --git a/install.sh b/install.sh index 7807158..bffb0b2 100755 --- a/install.sh +++ b/install.sh @@ -3,10 +3,17 @@ # -# OctoEverywhere for Klipper! +# OctoEverywhere for Klipper And Bambu Labs! # -# Use this script to install the plugin on a normal device or a Creality device, or to install the companion! +# Use this script to install the OctoEverywhere plugin for: +# OctoEverywhere for Klipper - Where this device is running Moonraker. +# OctoEverywhere for Creality - Where this device is a Creality device (Sonic Pad, K1, etc) +# OctoEverywhere Companion Plugin - Where this plugin will connect to Moonraker running on a different device on the same LAN +# OctoEverywhere Bambu Connect - Where this plugin will connect to a Bambu printer running on the save LAN. +# +# For a device running Klipper locally, use no arguments. # For a companion install, use the -companion argument. +# For a Bambu Connect install, use the -bambu argument. # # Simply run ./install.sh from the git repo root directory to get started! # @@ -318,11 +325,13 @@ cat << EOF @@@@@@@@@@@@@@@,,,,,,,,,,,,,,,,,,,,,@@@@@@@@@@@@@@ EOF log_blank +#log_header " OctoEverywhere For Klipper And Bambu Labs" log_important " OctoEverywhere For Klipper" log_blue " The 3D Printing Communities #1 Remote Access And AI Cloud Service" log_blank log_blank log_important "OctoEverywhere empowers the worldwide maker community with..." +#log_info " - Free & Unlimited Mainsail, Fluidd, And Bambu Labs Printers Remote Access" log_info " - Free & Unlimited Mainsail and Fluidd Remote Access" log_info " - Free & Unlimited Next-Gen AI Print Failure Detection" log_info " - Free Full Frame Rate & Full Resolution Webcam Streaming" @@ -381,9 +390,9 @@ cd ${OE_REPO_DIR} > /dev/null if [[ $IS_SONIC_PAD_OS -eq 1 ]] || [[ $IS_K1_OS -eq 1 ]] then # Creality OS only has a root user and we can't use sudo. - ${OE_ENV}/bin/python3 -B -m moonraker_installer ${PY_LAUNCH_JSON} + ${OE_ENV}/bin/python3 -B -m py_installer ${PY_LAUNCH_JSON} else - sudo ${OE_ENV}/bin/python3 -B -m moonraker_installer ${PY_LAUNCH_JSON} + sudo ${OE_ENV}/bin/python3 -B -m py_installer ${PY_LAUNCH_JSON} fi cd ${CURRENT_DIR} > /dev/null diff --git a/linux_host/__init__.py b/linux_host/__init__.py new file mode 100644 index 0000000..f00651b --- /dev/null +++ b/linux_host/__init__.py @@ -0,0 +1,3 @@ +# Need to make this a module + +# linux_host is a module that holds common logic for the different linux hosts. diff --git a/moonraker_octoeverywhere/config.py b/linux_host/config.py similarity index 75% rename from moonraker_octoeverywhere/config.py rename to linux_host/config.py index de6cda9..f04a20e 100644 --- a/moonraker_octoeverywhere/config.py +++ b/linux_host/config.py @@ -4,24 +4,34 @@ import configparser # This is what we use as our important settings config. +# This single config class is used for all of the plugin types, but not all of the values are used for each type. # It's a bit heavy handed with the lock and aggressive saving, but these # settings are important, and not accessed much. class Config: - # This can't change or all past plugins will fail. It's also used by the installer. + # This can't change or all past plugins will fail. ConfigFileName = "octoeverywhere.conf" - ServerSection = "server" + # + # Common To All Plugins + # + LoggingSection = "logging" + LogLevelKey = "log_level" + LogFileMaxSizeMbKey = "max_file_size_mb" + LogFileMaxCountKey = "max_file_count" + + # + # Used for the local Moonraker plugin and companions. + # RelaySection = "relay" RelayFrontEndPortKey = "frontend_port" # This field is shared with the installer, the installer can write this value. It the name can't change! RelayFrontEndTypeHintKey = "frontend_type_hint" # This field is shared with the installer, the installer can write this value. It the name can't change! - LoggingSection = "logging" - LogLevelKey = "log_level" - LogFileMaxSizeMbKey = "max_file_size_mb" - LogFileMaxCountKey = "max_file_count" + # + # Used for the local Moonraker plugin and companions. + # WebcamSection = "webcam" WebcamAutoSettings = "auto_settings_detection" WebcamNameToUseAsPrimary = "webcam_name_to_use_as_primary" @@ -31,6 +41,23 @@ class Config: WebcamFlipV = "flip_vertically" WebcamRotation = "rotate" + + # + # Used for both the companion and bambu connect plugins + # + SectionCompanion = "companion" + CompanionKeyIpOrHostname = "ip_or_hostname" + CompanionKeyPort = "port" + + + # + # Used only for Bambu Connect + # + SectionBambu = "bambu" + BambuAccessToken = "access_token" + BambuPrinterSn = "printer_serial_number" + + # This allows us to add comments into our config. # The objects must have two parts, first, a string they target. If the string is found, the comment will be inserted above the target string. This can be a section or value. # A string, which is the comment to be inserted. @@ -38,6 +65,10 @@ class Config: { "Target": RelayFrontEndPortKey, "Comment": "The port used for http relay. If your desired frontend runs on a different port, change this value. The OctoEverywhere plugin service needs to be restarted before changes will take effect."}, { "Target": RelayFrontEndTypeHintKey, "Comment": "A string only used by the UI to hint at what web interface this port is."}, { "Target": LogLevelKey, "Comment": "The active logging level. Valid values include: DEBUG, INFO, WARNING, or ERROR."}, + { "Target": CompanionKeyIpOrHostname, "Comment": "The IP or hostname this companion plugin will use to connect to Moonraker. The OctoEverywhere plugin service needs to be restarted before changes will take effect."}, + { "Target": CompanionKeyPort, "Comment": "The port this companion plugin will use to connect to Moonraker. The OctoEverywhere plugin service needs to be restarted before changes will take effect."}, + { "Target": BambuAccessToken, "Comment": "The access token to the Bambu printer. It can be found using the LCD screen on the printer, in the settings. The OctoEverywhere plugin service needs to be restarted before changes will take effect."}, + { "Target": BambuPrinterSn, "Comment": "The serial number of your Bambu printer. It can be found using this guide: https://wiki.bambulab.com/en/general/find-sn The OctoEverywhere plugin service needs to be restarted before changes will take effect."}, { "Target": WebcamNameToUseAsPrimary, "Comment": "This is the webcam name OctoEverywhere will use for Gadget AI, notifications, and such. This much match the camera 'Name' from your Mainsail of Fluidd webcam settings. The default value of 'Default' will pick whatever camera the system can find."}, { "Target": WebcamAutoSettings, "Comment": "Enables or disables auto webcam setting detection. If enabled, OctoEverywhere will find the webcam settings configured via the frontend (Fluidd, Mainsail, etc) and use them. Disable to manually set the values and have them not be overwritten."}, { "Target": WebcamStreamUrl, "Comment": "Webcam streaming URL. This can be a local relative path (ex: /webcam/?action=stream) or absolute http URL (ex: http://10.0.0.1:8080/webcam/?action=stream or http://webcam.local/webcam/?action=stream)"}, @@ -47,16 +78,17 @@ class Config: { "Target": WebcamRotation, "Comment": "Rotates the webcam image. Valid values are 0, 90, 180, or 270"}, ] + # The config lib we use doesn't support the % sign, even though it's valid .cfg syntax. # Since we save URLs into the config for the webcam, it's valid syntax to use a %20 and such, thus we should support it. PercentageStringReplaceString = "~~~PercentageSignPlaceholder~~~" - def __init__(self, klipperConfigPath) -> None: + def __init__(self, configDirPath:str) -> None: self.Logger = None # Define our config path # Note this path and name MUST STAY THE SAME because the installer PY script looks for this file. - self.OeConfigFilePath = os.path.join(klipperConfigPath, Config.ConfigFileName) + self.OeConfigFilePath = Config.GetConfigFilePath(configDirPath) # A lock to keep file access super safe self.ConfigLock = threading.Lock() self.Config = None @@ -65,6 +97,12 @@ def __init__(self, klipperConfigPath) -> None: self._LoadConfigIfNeeded_UnderLock() + # Returns the config file path given the config folder + @staticmethod + def GetConfigFilePath(configDirPath:str) -> str: + return os.path.join(configDirPath, Config.ConfigFileName) + + # Allows the logger to be set when it's created. def SetLogger(self, logger): self.Logger = logger @@ -79,6 +117,7 @@ def ReloadFromFile(self) -> None: # Gets a value from the config given the header and key. # If the value isn't set, the default value is returned and the default value is saved into the config. + # If the default value is None, the default will not be written into the config. def GetStr(self, section, key, defaultValue) -> str: with self.ConfigLock: # Ensure we have the config. @@ -100,10 +139,20 @@ def GetStr(self, section, key, defaultValue) -> str: # Gets a value from the config given the header and key. # If the value isn't set, the default value is returned and the default value is saved into the config. - def GetInt(self, section, key, defaultValue) -> int: + # If the default value is None, the default will not be written into the config. + def GetInt(self, section:str, key:str, defaultValue) -> int: # Use a try catch, so if a user sets an invalid value, it doesn't crash us. try: - return int(self.GetStr(section, key, str(defaultValue))) + # If None is passed as the default, don't str it. + if defaultValue is not None: + defaultValue = str(defaultValue) + + result = self.GetStr(section, key, defaultValue) + # If None is returned, don't int it, return None. + if result is None: + return None + + return int(str) except Exception as e: self.Logger.error("Config settings error! "+key+" failed to get as int. Resetting to default. "+str(e)) self.SetStr(section, key, str(defaultValue)) @@ -112,10 +161,21 @@ def GetInt(self, section, key, defaultValue) -> int: # Gets a value from the config given the header and key. # If the value isn't set, the default value is returned and the default value is saved into the config. + # If the default value is None, the default will not be written into the config. def GetBool(self, section, key, defaultValue) -> bool: # Use a try catch, so if a user sets an invalid value, it doesn't crash us. try: - strValue = self.GetStr(section, key, str(defaultValue)).lower() + # If None is passed as the default, don't str it. + if defaultValue is not None: + defaultValue = str(defaultValue) + strValue = self.GetStr(section, key, defaultValue) + + # If None is returned, don't bool it, return None. + if strValue is None: + return None + + # Match it to a bool value. + strValue = strValue.lower() if strValue == "false": return False elif strValue == "true": @@ -131,11 +191,13 @@ def GetBool(self, section, key, defaultValue) -> bool: # acceptable value list. If it's not, the default value is used. def GetStrIfInAcceptableList(self, section, key, defaultValue, acceptableValueList) -> str: existing = self.GetStr(section, key, defaultValue) - # Check the acceptable values - for v in acceptableValueList: - # If we match, this is a good value, return it. - if v.lower() == existing.lower(): - return existing + + if existing is not None: + # Check the acceptable values + for v in acceptableValueList: + # If we match, this is a good value, return it. + if v.lower() == existing.lower(): + return existing # The acceptable was not found. Set they key back to default. self.SetStr(section, key, defaultValue) @@ -144,8 +206,11 @@ def GetStrIfInAcceptableList(self, section, key, defaultValue, acceptableValueLi # The same as Get, but it makes sure the value is in a range. def GetIntIfInRange(self, section, key, defaultValue, lowerBoundInclusive, upperBoundInclusive) -> int: - existingStr = self.GetStr(section, key, str(defaultValue)) + # A default value of None is not allowed here. + if defaultValue is None: + raise Exception(f"A default value of none is not valid for int ranges. {section}:{key}") + existingStr = self.GetStr(section, key, str(defaultValue)) # Make sure the value is in range. try: existing = int(existingStr) diff --git a/moonraker_octoeverywhere/logger.py b/linux_host/logger.py similarity index 92% rename from moonraker_octoeverywhere/logger.py rename to linux_host/logger.py index 5058a7e..ecc1a8c 100644 --- a/moonraker_octoeverywhere/logger.py +++ b/linux_host/logger.py @@ -9,7 +9,7 @@ class LoggerInit: # Sets up and returns the main logger object @staticmethod - def GetLogger(config, klipperLogDir, logLevelOverride_CanBeNone) -> logging.Logger: + def GetLogger(config:Config, logDir:str, logLevelOverride_CanBeNone:str) -> logging.Logger: logger = logging.getLogger() # From the possible logging values, read the current value from the config. @@ -43,7 +43,7 @@ def GetLogger(config, klipperLogDir, logLevelOverride_CanBeNone) -> logging.Logg maxFileSizeBytes = config.GetIntIfInRange(Config.LoggingSection, Config.LogFileMaxSizeMbKey, 3, 1, 5000) * 1024 * 1024 maxFileCount = config.GetIntIfInRange(Config.LoggingSection, Config.LogFileMaxCountKey, 1, 1, 50) file = logging.handlers.RotatingFileHandler( - os.path.join(klipperLogDir, "octoeverywhere.log"), + os.path.join(logDir, "octoeverywhere.log"), maxBytes=maxFileSizeBytes, backupCount=maxFileCount) file.setFormatter(formatter) logger.addHandler(file) diff --git a/moonraker_octoeverywhere/secrets.py b/linux_host/secrets.py similarity index 95% rename from moonraker_octoeverywhere/secrets.py rename to linux_host/secrets.py index 6cf7ae6..e41684f 100644 --- a/moonraker_octoeverywhere/secrets.py +++ b/linux_host/secrets.py @@ -4,10 +4,8 @@ import configparser -from .config import Config - -# This class is very similar to the config class, but since the klipper config files are often backup -# in public places, the secrets are stored else where. +# This class is very similar to the config class, but since the klipper config files are often backup in public places, the secrets are stored else where. +# This is also used for the companion and the bambu host. class Secrets: # These must stay the same because our installer script requires on the format being as is! @@ -25,7 +23,7 @@ class Secrets: ] - def __init__(self, logger:logging.Logger, octoeverywhereStoragePath:str, config:Config) -> None: + def __init__(self, logger:logging.Logger, octoeverywhereStoragePath:str, moonrakerConfig = None) -> None: self.Logger = logger # Note this path and name MUST STAY THE SAME because the installer PY script looks for this file. @@ -41,7 +39,8 @@ def __init__(self, logger:logging.Logger, octoeverywhereStoragePath:str, config: # As of the creation of this class, 8/15/2023, we might need to migrate the printer id and private key to this file. # Do that before anything else is done. - self._DoConfigMigrationIfNeeded(config) + if moonrakerConfig is not None: + self._DoConfigMigrationIfNeeded(moonrakerConfig) # Returns the printer id if one exists, otherwise None. @@ -66,7 +65,7 @@ def SetPrivateKey(self, privateKey): # Migrates any old secrets from the config to the new location, if needed. # This must be called before the printer id or private key are accessed! - def _DoConfigMigrationIfNeeded(self, config:Config): + def _DoConfigMigrationIfNeeded(self, config): # Try to get the old values, if they exist. # We keep the old strings here, incase they ever change in new config updates. configServerSection = "server" diff --git a/linux_host/startup.py b/linux_host/startup.py new file mode 100644 index 0000000..4962ed3 --- /dev/null +++ b/linux_host/startup.py @@ -0,0 +1,89 @@ +import os +import sys +import json +import base64 +from enum import Enum + + +class ConfigDataTypes(Enum): + String = 1 + Path = 2 + Bool = 3 + +# +# Helper functions for the service startup. +# +class Startup: + + # A common error printing function. + def PrintErrorAndExit(self, msg:str): + print(f"\r\nPlugin Init Error - {msg}", file=sys.stderr) + print( "\r\nPlease contact support so we can fix this for you! support@octoeverywhere.com", file=sys.stderr) + sys.exit(1) + + + # Given the process args, this returns the json config. + def GetJsonFromArgs(self, argv): + # The config and settings path is passed as the first arg when the service runs. + # This allows us to run multiple services instances, each pointing at it's own config. + if len(argv) < 1: + self.PrintErrorAndExit("No program and json settings path passed to service") + + # The second arg should be a json string, which has all of our params. + if len(argv) < 2: + self.PrintErrorAndExit("No json settings path passed to service") + + # Try to parse the config + jsonConfigStr = None + try: + # The args are passed as a urlbase64 encoded string, to prevent issues with passing some chars as args. + argsJsonBase64 = argv[1] + jsonConfigStr = base64.urlsafe_b64decode(bytes(argsJsonBase64, "utf-8")).decode("utf-8") + print("Loading Service Config: "+jsonConfigStr) + return json.loads(jsonConfigStr) + except Exception as e: + self.PrintErrorAndExit("Failed to get json from cmd args. "+str(e)) + return None + + + # If there was a dev config passed, this parses it and returns the json object. + def GetDevConfigIfAvailable(self, argv): + try: + if len(argv) > 2: + devConfigJson = json.loads(argv[2]) + print("Using dev config: "+argv[2]) + return devConfigJson + except Exception as e: + self.PrintErrorAndExit(f"Exception while DEV CONFIG. Error:{str(e)}, Config: {argv[2]}") + return None + + + # A helper to get a specific value from the json config. + # oldVarName allows us to stay compat with older installs. + def GetConfigVarAndValidate(self, jsonConfig, varName:str, dataType:ConfigDataTypes, oldVarName:str = None): + var = None + if varName in jsonConfig: + var = jsonConfig[varName] + elif oldVarName in jsonConfig: + var = jsonConfig[oldVarName] + else: + raise Exception(f"{varName} isn't found in the json jsonConfig.") + + if var is None: + raise Exception(f"{varName} returned None when parsing json jsonConfig.") + + if dataType == ConfigDataTypes.String or dataType == ConfigDataTypes.Path: + var = str(var) + if len(var) == 0: + raise Exception(f"{varName} is an empty string.") + + if dataType == ConfigDataTypes.Path: + if os.path.exists(var) is False: + raise Exception(f"{varName} is a path, but the path wasn't found.") + + elif dataType == ConfigDataTypes.Bool: + var = bool(var) + + else: + raise Exception(f"{varName} has an invalid jsonConfig data type. {dataType}") + return var diff --git a/moonraker_octoeverywhere/version.py b/linux_host/version.py similarity index 100% rename from moonraker_octoeverywhere/version.py rename to linux_host/version.py diff --git a/moonraker_installer/DiscoveryObserver.py b/moonraker_installer/DiscoveryObserver.py deleted file mode 100644 index 047bb34..0000000 --- a/moonraker_installer/DiscoveryObserver.py +++ /dev/null @@ -1,110 +0,0 @@ -import os - -from .Logging import Logger -from .Context import Context -from .Util import Util -from .ObserverConfigFile import ObserverConfigFile - - -# This class does the same function as the Discovery class, but for the observer plugin setup. -class DiscoveryObserver: - - # This is the base data folder name that will be used, the plugin id suffix will be added to end of it. - # The folders will always be in the user's home path. - c_ObserverPluginDataRootFolder_Lower = "octoeverywhere-companion-" - # The legacy name, only used to find existing folders. - c_ObserverPluginDataRootFolder_old_Lower = ".octoeverywhere-observer-" - - def ObserverDiscovery(self, context:Context): - Logger.Debug("Starting observer discovery.") - - # Look for existing observer data installs. - existingObserverFolders = [] - # Sort so the folder we find are ordered from 1-... This makes the selection process nicer, since the ID == selection. - fileAndDirList = sorted(os.listdir(context.UserHomePath)) - for fileOrDirName in fileAndDirList: - if fileOrDirName.lower().startswith(DiscoveryObserver.c_ObserverPluginDataRootFolder_Lower) or fileOrDirName.lower().startswith(DiscoveryObserver.c_ObserverPluginDataRootFolder_old_Lower): - existingObserverFolders.append(fileOrDirName) - Logger.Debug(f"Found existing data folder: {fileOrDirName}") - - # If there's an existing folders, ask the user if they want to use them. - if len(existingObserverFolders) > 0: - count = 1 - Logger.Blank() - Logger.Header("Existing OctoEverywhere Observer Plugins Found") - Logger.Blank() - Logger.Info( "If you want to update or re-setup an instance, select instance id.") - Logger.Info( " - or - ") - Logger.Info( "If you want to install a new instance, select 'n'.") - Logger.Blank() - Logger.Info("Options:") - for folder in existingObserverFolders: - instanceId = self._GetObserverIdFromFolderName(folder) - # Try to parse the config, if there is one and it's valid. - ip, port = ObserverConfigFile.TryToParseConfig(ObserverConfigFile.GetConfigFilePathFromDataPath(os.path.join(context.UserHomePath, folder))) - if ip is None and port is None: - Logger.Info(f" {count}) Instance Id {instanceId} - Path: {folder}") - else: - Logger.Info(f" {count}) Instance Id {instanceId} - {ip}:{port}") - count += 1 - Logger.Info(" n) Setup a new observer plugin instance") - Logger.Blank() - Logger.Blank() - # Ask the user which number they want. - responseInt = -1 - isFirstPrint = True - while True: - try: - if isFirstPrint: - isFirstPrint = False - else: - Logger.Warn( "If you need help, contact us! https://octoeverywhere.com/support") - response = input("Enter an instance id or 'n': ") - response = response.lower().strip() - # If the response is n, fall through. - if response == "n": - break - # Parse the input and -1 it, so it aligns with the array length. - tempInt = int(response.lower().strip()) - 1 - if tempInt >= 0 and tempInt < len(existingObserverFolders): - responseInt = tempInt - break - Logger.Warn("Invalid number selection, try again.") - except Exception as _: - Logger.Warn("Invalid input, try again.") - - # If there is a response, the user selected an instance. - if responseInt != -1: - # Use this instance - self._SetupContextFromVars(context, existingObserverFolders[responseInt]) - Logger.Info(f"Existing observer instance selected. Path: {context.ObserverDataPath}, Id: {context.ObserverInstanceId}") - return - - # Create a new instance path. Either there is no existing data path or the user wanted to create a new one. - # Since we have all of the data paths, we will make this new instance id be the count + 1. - newId = str(len(existingObserverFolders) + 1) - self._SetupContextFromVars(context, f"{DiscoveryObserver.c_ObserverPluginDataRootFolder_Lower}{newId}") - Logger.Info(f"Creating a new Observer plugin data path. Path: {context.ObserverDataPath}, Id: {context.ObserverInstanceId}") - return - - - def _SetupContextFromVars(self, context:Context, folderName:str): - # First, ensure we can parse the id and set it. - context.ObserverInstanceId = self._GetObserverIdFromFolderName(folderName) - - # Make the full path - context.ObserverDataPath = os.path.join(context.UserHomePath, folderName) - - # Ensure the file exists and we have permissions - Util.EnsureDirExists(context.ObserverDataPath, context, True) - - - def _GetObserverIdFromFolderName(self, folderName:str): - folderName_lower = folderName.lower() - # If we can find either of the names, return everything after the prefix, aka the instance id. - if folderName_lower.startswith(DiscoveryObserver.c_ObserverPluginDataRootFolder_Lower) is True: - return folderName_lower[len(DiscoveryObserver.c_ObserverPluginDataRootFolder_Lower):] - if folderName_lower.startswith(DiscoveryObserver.c_ObserverPluginDataRootFolder_old_Lower) is True: - return folderName_lower[len(DiscoveryObserver.c_ObserverPluginDataRootFolder_old_Lower):] - Logger.Error(f"We tried to get an observer id from a non-observer data folder. {folderName}") - raise Exception("We tried to get an observer id from a non-observer data folder") diff --git a/moonraker_installer/ObserverConfigFile.py b/moonraker_installer/ObserverConfigFile.py deleted file mode 100644 index a820812..0000000 --- a/moonraker_installer/ObserverConfigFile.py +++ /dev/null @@ -1,68 +0,0 @@ - -import configparser -import os - -from .Logging import Logger -from .Util import Util -from .Context import Context - -class ObserverConfigFile: - - # These sections and keys are shared with the moonraker plugin code, so we can't change them. - c_SectionMoonraker = "moonraker" - c_KeyIpOrHostname = "ip_or_hostname" - c_KeyPort = "port" - - - @staticmethod - def GetConfigFolderPathFromDataPath(observerDataPath:str): - # This path is shared with the plugin, so it can't be changed. - return os.path.join(observerDataPath, "config") - - - @staticmethod - def GetConfigFilePathFromDataPath(observerDataPath:str): - # This file name is shared with the plugin, so it can't be changed. - return os.path.join(ObserverConfigFile.GetConfigFolderPathFromDataPath(observerDataPath), "octoeverywhere-observer.cfg") - - - # Returns the (ip:str, port:str) if the config can be parsed. Otherwise (None, None) - @staticmethod - def TryToParseConfig(configPath:str): - if os.path.exists(configPath): - try: - config = configparser.ConfigParser(allow_no_value=True, strict=False) - config.read(configPath) - if config.has_section(ObserverConfigFile.c_SectionMoonraker): - if ObserverConfigFile.c_KeyIpOrHostname in config[ObserverConfigFile.c_SectionMoonraker].keys() and ObserverConfigFile.c_KeyPort in config[ObserverConfigFile.c_SectionMoonraker].keys(): - ip = config[ObserverConfigFile.c_SectionMoonraker][ObserverConfigFile.c_KeyIpOrHostname] - port = config[ObserverConfigFile.c_SectionMoonraker][ObserverConfigFile.c_KeyPort] - if len(ip) > 0: - portInt = int(port) - if portInt > 0 and portInt < 65535: - return (ip, port) - except Exception as e: - Logger.Debug(f"Failed to parse plugin observer config: {configPath}; " + str(e)) - return (None, None) - - - # Creates or uses an existing config, updates the ip and port. - @staticmethod - def WriteIpAndPort(context:Context, configPath:str, ip:str, port:str): - try: - # Ensure the dir exits. - Util.EnsureDirExists(Util.GetParentDirectory(configPath), context, True) - # Read the file, if there is one. - config = configparser.ConfigParser(allow_no_value=True, strict=False) - if os.path.exists(configPath): - config.read(configPath) - if config.has_section(ObserverConfigFile.c_SectionMoonraker) is False: - config.add_section(ObserverConfigFile.c_SectionMoonraker) - config[ObserverConfigFile.c_SectionMoonraker][ObserverConfigFile.c_KeyIpOrHostname] = ip - config[ObserverConfigFile.c_SectionMoonraker][ObserverConfigFile.c_KeyPort] = port - with open(configPath, 'w', encoding="utf-8") as f: - config.write(f) - return True - except Exception as e: - Logger.Error("Failed to write observer config. "+str(e)) - return False diff --git a/moonraker_installer/__init__.py b/moonraker_installer/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/moonraker_octoeverywhere/__main__.py b/moonraker_octoeverywhere/__main__.py index e7ae072..44b5090 100644 --- a/moonraker_octoeverywhere/__main__.py +++ b/moonraker_octoeverywhere/__main__.py @@ -1,125 +1,69 @@ -import os import sys import json -import base64 -from enum import Enum -from .moonrakerhost import MoonrakerHost - -# -# Helper functions for config parsing and validation. -# -class ConfigDataTypes(Enum): - String = 1 - Path = 2 - Bool = 3 - -def _GetConfigVarAndValidate(config, varName:str, dataType:ConfigDataTypes): - if varName not in config: - raise Exception(f"{varName} isn't found in the json config.") - var = config[varName] - - if var is None: - raise Exception(f"{varName} returned None when parsing json config.") - - if dataType == ConfigDataTypes.String or dataType == ConfigDataTypes.Path: - var = str(var) - if len(var) == 0: - raise Exception(f"{varName} is an empty string.") - - if dataType == ConfigDataTypes.Path: - if os.path.exists(var) is False: - raise Exception(f"{varName} is a path, but the path wasn't found.") - - elif dataType == ConfigDataTypes.Bool: - var = bool(var) - - else: - raise Exception(f"{varName} has an invalid config data type. {dataType}") - return var - -# -# Helper for errors. -# -def _PrintErrorAndExit(msg:str): - print(f"\r\nPlugin Init Error - {msg}", file=sys.stderr) - print( "\r\nPlease contact support so we can fix this for you! support@octoeverywhere.com", file=sys.stderr) - sys.exit(1) +from linux_host.startup import Startup +from linux_host.startup import ConfigDataTypes +from .moonrakerhost import MoonrakerHost if __name__ == '__main__': - # The config and settings path is passed as the first arg when the service runs. - # This allows us to run multiple services instances, each pointing at it's own config. - if len(sys.argv) < 1: - _PrintErrorAndExit("No program and json settings path passed to service") - # The second arg should be a json string, which has all of our params. - if len(sys.argv) < 2: - _PrintErrorAndExit("No json settings path passed to service") + # This is a helper class, to keep the startup logic common. + s = Startup() # Try to parse the config - jsonConfigStr = None + jsonConfig = None try: - # The args are passed as a urlbase64 encoded string, to prevent issues with passing some chars as args. - argsJsonBase64 = sys.argv[1] - jsonConfigStr = base64.urlsafe_b64decode(bytes(argsJsonBase64, "utf-8")).decode("utf-8") - print("Loading Service Config: "+jsonConfigStr) - config = json.loads(jsonConfigStr) + # Get the json from the process args. + jsonConfig = s.GetJsonFromArgs(sys.argv) # # 1) Parse the common, required args. # - ServiceName = _GetConfigVarAndValidate(config, "ServiceName", ConfigDataTypes.String) - VirtualEnvPath = _GetConfigVarAndValidate(config, "VirtualEnvPath", ConfigDataTypes.Path) - RepoRootFolder = _GetConfigVarAndValidate(config, "RepoRootFolder", ConfigDataTypes.Path) - KlipperConfigFolder = _GetConfigVarAndValidate(config, "KlipperConfigFolder", ConfigDataTypes.Path) - KlipperLogFolder = _GetConfigVarAndValidate(config, "KlipperLogFolder", ConfigDataTypes.Path) - LocalFileStoragePath = _GetConfigVarAndValidate(config, "LocalFileStoragePath", ConfigDataTypes.Path) + ServiceName = s.GetConfigVarAndValidate(jsonConfig, "ServiceName", ConfigDataTypes.String) + VirtualEnvPath = s.GetConfigVarAndValidate(jsonConfig, "VirtualEnvPath", ConfigDataTypes.Path) + RepoRootFolder = s.GetConfigVarAndValidate(jsonConfig, "RepoRootFolder", ConfigDataTypes.Path) + LocalFileStoragePath = s.GetConfigVarAndValidate(jsonConfig, "LocalFileStoragePath", ConfigDataTypes.Path) + # These var names changed to support other plugin types like Bambu, but we must keep them around for older installs. + KlipperConfigFolder = s.GetConfigVarAndValidate(jsonConfig, "ConfigFolder", ConfigDataTypes.Path, "KlipperConfigFolder") + KlipperLogFolder = s.GetConfigVarAndValidate(jsonConfig, "LogFolder", ConfigDataTypes.Path, "KlipperLogFolder") # - # 2) Parse the IsObserver flag, this will determine which other vars are required. - # Note that for older plugin installs, the IsObserver flag won't exist, implying False. + # 2) Parse the IsCompanion flag, this will determine which other vars are required. + # Note that for older plugin installs, the IsCompanion flag won't exist, implying False. # - IsObserver = False - if "IsObserver" in config: - IsObserver = config["IsObserver"] - IsObserver = bool(IsObserver) + IsCompanion = False + if "IsCompanion" in jsonConfig: + IsCompanion = jsonConfig["IsCompanion"] + IsCompanion = bool(IsCompanion) # - # 3) Now parse the required vars based on the IsObserver flag state. + # 3) Now parse the required vars based on the IsCompanion flag state. # MoonrakerConfigFile = None - ObserverConfigFilePath = None - ObserverInstanceIdStr = None + #CompanionInstanceIdStr = None - if IsObserver: - ObserverConfigFilePath = _GetConfigVarAndValidate(config, "ObserverConfigFilePath", ConfigDataTypes.Path) - ObserverInstanceIdStr = _GetConfigVarAndValidate(config, "ObserverInstanceIdStr", ConfigDataTypes.String) + if IsCompanion: + # We don't use this right now, but we have it if we need it. + #CompanionInstanceIdStr = s.GetConfigVarAndValidate(jsonConfig, "CompanionInstanceIdStr", ConfigDataTypes.String) + pass else: - MoonrakerConfigFile = _GetConfigVarAndValidate(config, "MoonrakerConfigFile", ConfigDataTypes.Path) + MoonrakerConfigFile = s.GetConfigVarAndValidate(jsonConfig, "MoonrakerConfigFile", ConfigDataTypes.Path) except Exception as e: - _PrintErrorAndExit(f"Exception while loading json config. Error:{str(e)}, Config: {jsonConfigStr}") + s.PrintErrorAndExit(f"Exception while loading json config. Error:{str(e)} Config:{json.dumps(jsonConfig)}") # For debugging, we also allow an optional dev object to be passed. - devConfig_CanBeNone = None - try: - if len(sys.argv) > 2: - devConfig_CanBeNone = json.loads(sys.argv[2]) - print("Using dev config: "+sys.argv[2]) - except Exception as e: - _PrintErrorAndExit(f"Exception while DEV CONFIG. Error:{str(e)}, Config: {sys.argv[2]}") + devConfig_CanBeNone = s.GetDevConfigIfAvailable(sys.argv) # Run! try: # Create and run the main host! host = MoonrakerHost(KlipperConfigFolder, KlipperLogFolder, devConfig_CanBeNone) - host.RunBlocking(KlipperConfigFolder, IsObserver, LocalFileStoragePath, ServiceName, VirtualEnvPath, RepoRootFolder, - MoonrakerConfigFile, - ObserverConfigFilePath, ObserverInstanceIdStr, - devConfig_CanBeNone) + host.RunBlocking(KlipperConfigFolder, IsCompanion, LocalFileStoragePath, ServiceName, VirtualEnvPath, RepoRootFolder, + MoonrakerConfigFile, devConfig_CanBeNone) except Exception as e: - _PrintErrorAndExit(f"Exception leaked from main moonraker host class. Error:{str(e)}") + s.PrintErrorAndExit(f"Exception leaked from main moonraker host class. Error:{str(e)}") # If we exit here, it's due to an error, since RunBlocking should be blocked forever. sys.exit(1) diff --git a/moonraker_octoeverywhere/moonrakerclient.py b/moonraker_octoeverywhere/moonrakerclient.py index e3bdf40..b2f9025 100644 --- a/moonraker_octoeverywhere/moonrakerclient.py +++ b/moonraker_octoeverywhere/moonrakerclient.py @@ -5,16 +5,17 @@ import queue import logging import math - import configparser -from octoeverywhere.compat import Compat +from octoeverywhere.compat import Compat from octoeverywhere.sentry import Sentry from octoeverywhere.websocketimpl import Client from octoeverywhere.notificationshandler import NotificationsHandler -from .moonrakercredentailmanager import MoonrakerCredentialManager + +from linux_host.config import Config + from .filemetadatacache import FileMetadataCache -from .observerconfigfile import ObserverConfigFile +from .moonrakercredentailmanager import MoonrakerCredentialManager # The response object for a json rpc request. # Contains information on the state, and if successful, the result. @@ -65,8 +66,8 @@ class MoonrakerClient: WebSocketMessageDebugging = False @staticmethod - def Init(logger, isObserverMode:bool, moonrakerConfigFilePath:str, observerConfigPath:str, printerId, connectionStatusHandler, pluginVersionStr): - MoonrakerClient._Instance = MoonrakerClient(logger, isObserverMode, moonrakerConfigFilePath, observerConfigPath, printerId, connectionStatusHandler, pluginVersionStr) + def Init(logger, config, moonrakerConfigFilePath:str, printerId:str, connectionStatusHandler, pluginVersionStr:str): + MoonrakerClient._Instance = MoonrakerClient(logger, config, moonrakerConfigFilePath, printerId, connectionStatusHandler, pluginVersionStr) @staticmethod @@ -74,11 +75,10 @@ def Get(): return MoonrakerClient._Instance - def __init__(self, logger:logging.Logger, isObserverMode:bool, moonrakerConfigFilePath:str, observerConfigPath:str, printerId:str, connectionStatusHandler, pluginVersionStr:str) -> None: + def __init__(self, logger:logging.Logger, config:Config, moonrakerConfigFilePath:str, printerId:str, connectionStatusHandler, pluginVersionStr:str) -> None: self.Logger = logger - self.IsObserverMode = isObserverMode + self.Config = config self.MoonrakerConfigFilePath = moonrakerConfigFilePath - self.ObserverConfigPath = observerConfigPath self.MoonrakerHostAndPort = "127.0.0.1:7125" self.PrinterId = printerId self.ConnectionStatusHandler = connectionStatusHandler @@ -143,12 +143,8 @@ def StartRunningIfNotAlready(self, octoKey:str) -> None: # value in our settings, which could change. This is called by the Websocket and then the result is saved in the class # This is so every http call doesn't have to read the file, but as long as the WS is connected, we know the address is correct. def _UpdateMoonrakerHostAndPort(self) -> None: - # Ensure we have a file. For now, this is required. - if self.IsObserverMode: - if os.path.exists(self.ObserverConfigPath) is False: - self.Logger.error("Moonraker client failed to find a observer config. Re-run the ./install.sh script from the OctoEverywhere repo to update the path.") - raise Exception("No observer config file found") - else: + # If we aren't in companion mode, ensure there's a valid moonraker config file on disk + if Compat.IsCompanionMode() is False: if os.path.exists(self.MoonrakerConfigFilePath) is False: self.Logger.error("Moonraker client failed to find a moonraker config. Re-run the ./install.sh script from the OctoEverywhere repo to update the path.") raise Exception("No config file found") @@ -167,11 +163,12 @@ def GetMoonrakerHostAndPortFromConfig(self): currentPortInt = 7125 currentHostStr = "0.0.0.0" try: - # If we are in observer mode, we need to use the observer config to find the remote moonraker details. - if Compat.IsObserverMode(): - ip, portStr = ObserverConfigFile.Get().TryToGetIpAndPortStr() + # If we are in companion mode, we pull the moonraker connection details from the config. + if Compat.IsCompanionMode(): + ip = self.Config.GetStr(Config.SectionCompanion, Config.CompanionKeyIpOrHostname, None) + portStr = self.Config.GetStr(Config.SectionCompanion, Config.CompanionKeyPort, None) if ip is None or portStr is None: - self.Logger.error("Failed to get observer moonraker details from observer config.") + self.Logger.error("Failed to get companion moonraker details from config.") return (currentHostStr, currentPortInt) return (ip, int(portStr)) diff --git a/moonraker_octoeverywhere/moonrakercredentailmanager.py b/moonraker_octoeverywhere/moonrakercredentailmanager.py index 67174dd..7d17228 100644 --- a/moonraker_octoeverywhere/moonrakercredentailmanager.py +++ b/moonraker_octoeverywhere/moonrakercredentailmanager.py @@ -31,8 +31,8 @@ class MoonrakerCredentialManager: @staticmethod - def Init(logger, moonrakerConfigFilePath:str, isObserverMode:bool): - MoonrakerCredentialManager._Instance = MoonrakerCredentialManager(logger, moonrakerConfigFilePath, isObserverMode) + def Init(logger, moonrakerConfigFilePath:str, isCompanionMode:bool): + MoonrakerCredentialManager._Instance = MoonrakerCredentialManager(logger, moonrakerConfigFilePath, isCompanionMode) @staticmethod @@ -40,16 +40,16 @@ def Get(): return MoonrakerCredentialManager._Instance - def __init__(self, logger:logging.Logger, moonrakerConfigFilePath:str, isObserverMode:bool): + def __init__(self, logger:logging.Logger, moonrakerConfigFilePath:str, isCompanionMode:bool): self.Logger = logger self.MoonrakerConfigFilePath = moonrakerConfigFilePath - self.IsObserverMode = isObserverMode + self.IsCompanionMode = isCompanionMode def TryToGetApiKey(self) -> str or None: - # If this is an observer plugin, we dont' have the moonraker config file nor can we access the UNIX socket. - if self.IsObserverMode: - self.Logger.info("OctoEverywhere Companion Plugins dont' support Moonraker setups with auth.") + # If this is an companion plugin, we dont' have the moonraker config file nor can we access the UNIX socket. + if self.IsCompanionMode: + self.Logger.info("OctoEverywhere Companion Plugins don't support Moonraker setups with auth.") return None # First, we need to find the unix socket to connect to diff --git a/moonraker_octoeverywhere/moonrakerhost.py b/moonraker_octoeverywhere/moonrakerhost.py index 3dbccfc..8c30b0a 100644 --- a/moonraker_octoeverywhere/moonrakerhost.py +++ b/moonraker_octoeverywhere/moonrakerhost.py @@ -14,10 +14,11 @@ from octoeverywhere.localip import LocalIpHelper from octoeverywhere.compat import Compat -from .config import Config -from .secrets import Secrets -from .version import Version -from .logger import LoggerInit +from linux_host.config import Config +from linux_host.secrets import Secrets +from linux_host.version import Version +from linux_host.logger import LoggerInit + from .smartpause import SmartPause from .uipopupinvoker import UiPopupInvoker from .systemconfigmanager import SystemConfigManager @@ -30,7 +31,6 @@ from .moonrakercredentailmanager import MoonrakerCredentialManager from .filemetadatacache import FileMetadataCache from .uiinjector import UiInjector -from .observerconfigfile import ObserverConfigFile # This file is the main host for the moonraker service. class MoonrakerHost: @@ -65,9 +65,8 @@ def __init__(self, klipperConfigDir, klipperLogDir, devConfig_CanBeNone) -> None raise - def RunBlocking(self, klipperConfigDir, isObserverMode, localStorageDir, serviceName, pyVirtEnvRoot, repoRoot, - moonrakerConfigFilePath, # Will be None in Observer mode - observerConfigFilePath, observerInstanceIdStr, # Will be None in NOT Observer mode + def RunBlocking(self, klipperConfigDir, isCompanionMode, localStorageDir, serviceName, pyVirtEnvRoot, repoRoot, + moonrakerConfigFilePath, # Will be None in Companion mode devConfig_CanBeNone): # Do all of this in a try catch, so we can log any issues before exiting try: @@ -75,15 +74,15 @@ def RunBlocking(self, klipperConfigDir, isObserverMode, localStorageDir, service self.Logger.info("#### OctoEverywhere Starting #####") self.Logger.info("##################################") - # Set observer mode flag as soon as we know it. - Compat.SetIsObserverMode(isObserverMode) + # Set companion mode flag as soon as we know it. + Compat.SetIsCompanionMode(isCompanionMode) # Find the version of the plugin, this is required and it will throw if it fails. pluginVersionStr = Version.GetPluginVersion(repoRoot) self.Logger.info("Plugin Version: %s", pluginVersionStr) # This logic only works if running locally. - if not isObserverMode: + if not isCompanionMode: # Before we do this first time setup, make sure our config files are in place. This is important # because if this fails it will throw. We don't want to let the user complete the install setup if things # with the update aren't working. @@ -93,9 +92,6 @@ def RunBlocking(self, klipperConfigDir, isObserverMode, localStorageDir, service # As of 8/15/2023, we don't store any sensitive things in teh config file, since all config files are sometimes backed up publicly. self.Secrets = Secrets(self.Logger, localStorageDir, self.Config) - # Always init the observer config file class, even if we aren't in observer mode, it handles denying requests. - ObserverConfigFile.Init(self.Logger, observerConfigFilePath) - # Now, detect if this is a new instance and we need to init our global vars. If so, the setup script will be waiting on this. self.DoFirstTimeSetupIfNeeded(klipperConfigDir, serviceName) @@ -123,7 +119,7 @@ def RunBlocking(self, klipperConfigDir, isObserverMode, localStorageDir, service self.MoonrakerDatabase = MoonrakerDatabase(self.Logger, printerId, pluginVersionStr) # Setup the credential manager. - MoonrakerCredentialManager.Init(self.Logger, moonrakerConfigFilePath, isObserverMode) + MoonrakerCredentialManager.Init(self.Logger, moonrakerConfigFilePath, isCompanionMode) # Setup the http requester. We default to port 80 and assume the frontend can be found there. # TODO - parse nginx to see what front ends exist and make them switchable @@ -134,12 +130,13 @@ def RunBlocking(self, klipperConfigDir, isObserverMode, localStorageDir, service OctoHttpRequest.SetLocalHttpProxyIsHttps(False) OctoHttpRequest.SetLocalOctoPrintPort(frontendPort) - # If we are in observer mode, we need to update the local address to be the other local remote. - if isObserverMode: - (ipOrHostnameStr, portStr) = ObserverConfigFile.Get().TryToGetIpAndPortStr() + # If we are in companion mode, we need to update the local address to be the other local remote. + if isCompanionMode: + ipOrHostnameStr = self.Config.GetStr(Config.SectionCompanion, Config.CompanionKeyIpOrHostname, None) + portStr = self.Config.GetStr(Config.SectionCompanion, Config.CompanionKeyPort, None) if ipOrHostnameStr is None or portStr is None: - self.Logger.error("We are in observer mode but we can't get the ip and port from the observer config file.") - raise Exception("Failed to read observer config file.") + self.Logger.error("We are in companion mode but we can't get the ip and port from the companion config file.") + raise Exception("Failed to read companion config file.") OctoHttpRequest.SetLocalHostAddress(ipOrHostnameStr) # TODO - this could be an host name, not an IP. That might be a problem? LocalIpHelper.SetLocalIpOverride(ipOrHostnameStr) @@ -159,7 +156,7 @@ def RunBlocking(self, klipperConfigDir, isObserverMode, localStorageDir, service # When everything is setup, start the moonraker client object. # This also creates the Notifications Handler and Gadget objects. # This doesn't start the moon raker connection, we don't do that until OE connects. - MoonrakerClient.Init(self.Logger, isObserverMode, moonrakerConfigFilePath, observerConfigFilePath, printerId, self, pluginVersionStr) + MoonrakerClient.Init(self.Logger, self.Config, moonrakerConfigFilePath, printerId, self, pluginVersionStr) # Init our file meta data cache helper FileMetadataCache.Init(self.Logger, MoonrakerClient.Get()) @@ -182,7 +179,7 @@ def RunBlocking(self, klipperConfigDir, isObserverMode, localStorageDir, service OctoEverywhereWsUri = HostCommon.c_OctoEverywhereOctoClientWsUri if DevLocalServerAddress_CanBeNone is not None: OctoEverywhereWsUri = "ws://"+DevLocalServerAddress_CanBeNone+"/octoclientws" - oe = OctoEverywhere(OctoEverywhereWsUri, printerId, privateKey, self.Logger, UiPopupInvoker(self.Logger), self, pluginVersionStr, ServerHost.Moonraker, isObserverMode) + oe = OctoEverywhere(OctoEverywhereWsUri, printerId, privateKey, self.Logger, UiPopupInvoker(self.Logger), self, pluginVersionStr, ServerHost.Moonraker, isCompanionMode) oe.RunBlocking() except Exception as e: Sentry.Exception("!! Exception thrown out of main host run function.", e) diff --git a/moonraker_octoeverywhere/moonrakerwebcamhelper.py b/moonraker_octoeverywhere/moonrakerwebcamhelper.py index bcb9619..8db1b13 100644 --- a/moonraker_octoeverywhere/moonrakerwebcamhelper.py +++ b/moonraker_octoeverywhere/moonrakerwebcamhelper.py @@ -8,7 +8,8 @@ from octoeverywhere.sentry import Sentry from octoeverywhere.webcamhelper import WebcamSettingItem, WebcamHelper -from .config import Config +from linux_host.config import Config + from .moonrakerclient import MoonrakerClient from .moonrakerclient import JsonRpcResponse diff --git a/moonraker_octoeverywhere/observerconfigfile.py b/moonraker_octoeverywhere/observerconfigfile.py deleted file mode 100644 index a7f9e2b..0000000 --- a/moonraker_octoeverywhere/observerconfigfile.py +++ /dev/null @@ -1,56 +0,0 @@ - -import configparser -import logging -import os - -from octoeverywhere.compat import Compat - -class ObserverConfigFile: - - # These sections and keys are shared with the installer plugin code, so we can't change them. - c_SectionMoonraker = "moonraker" - c_KeyIpOrHostname = "ip_or_hostname" - c_KeyPort = "port" - - # The static instance. - _Instance = None - - - @staticmethod - def Init(logger:logging.Logger, observerConfigFile:str): - ObserverConfigFile._Instance = ObserverConfigFile(logger, observerConfigFile) - - - def __init__(self, logger:logging.Logger, observerConfigFile:str) -> None: - self.Logger = logger - # Note this will be None if we aren't in observer mode. - self.ObserverConfigFile = observerConfigFile - - - @staticmethod - def Get(): - return ObserverConfigFile._Instance - - - # Returns the (ip:str, port:str) if the config can be parsed. Otherwise (None, None) - def TryToGetIpAndPortStr(self): - if not Compat.IsObserverMode(): - self.Logger.error("Observer config file was attempted to be accessed without being in observer mode.") - return (None, None) - if not os.path.exists(self.ObserverConfigFile): - self.Logger.error("No Observer config file was set into the observer config class.") - return (None, None) - try: - config = configparser.ConfigParser(allow_no_value=True, strict=False) - config.read(self.ObserverConfigFile) - if config.has_section(ObserverConfigFile.c_SectionMoonraker): - if ObserverConfigFile.c_KeyIpOrHostname in config[ObserverConfigFile.c_SectionMoonraker].keys() and ObserverConfigFile.c_KeyPort in config[ObserverConfigFile.c_SectionMoonraker].keys(): - ip = config[ObserverConfigFile.c_SectionMoonraker][ObserverConfigFile.c_KeyIpOrHostname] - port = config[ObserverConfigFile.c_SectionMoonraker][ObserverConfigFile.c_KeyPort] - if len(ip) > 0: - portInt = int(port) - if portInt > 0 and portInt < 65535: - return (ip, port) - except Exception as e: - self.Logger.warn(f"Failed to parse plugin observer config: {self.ObserverConfigFile}; " + str(e)) - return (None, None) diff --git a/octoeverywhere/commandhandler.py b/octoeverywhere/commandhandler.py index a2d8355..b9dcf3a 100644 --- a/octoeverywhere/commandhandler.py +++ b/octoeverywhere/commandhandler.py @@ -164,7 +164,7 @@ def ListWebcams(self): webcamSettingsItems = WebcamHelper.Get().ListWebcams() if webcamSettingsItems is None: webcamSettingsItems = [] - # We need to convert the objects into a dic to seralize. + # We need to convert the objects into a dic to serialize. webcams = [] for i in webcamSettingsItems: wc = {} diff --git a/octoeverywhere/compat.py b/octoeverywhere/compat.py index 12c8752..f46a30f 100644 --- a/octoeverywhere/compat.py +++ b/octoeverywhere/compat.py @@ -6,7 +6,8 @@ class Compat: _IsOctoPrintHost = False _IsMoonrakerHost = False - _IsObserverMode = False + _IsCompanionMode = False + _IsBambu = False @staticmethod def IsOctoPrint() -> bool: return Compat._IsOctoPrintHost @@ -14,8 +15,8 @@ def IsOctoPrint() -> bool: def IsMoonraker() -> bool: return Compat._IsMoonrakerHost @staticmethod - def IsObserverMode() -> bool: - return Compat._IsObserverMode + def IsCompanionMode() -> bool: + return Compat._IsCompanionMode @staticmethod def SetIsOctoPrint(b): Compat._IsOctoPrintHost = b @@ -23,8 +24,11 @@ def SetIsOctoPrint(b): def SetIsMoonraker(b): Compat._IsMoonrakerHost = b @staticmethod - def SetIsObserverMode(b): - Compat._IsObserverMode = b + def SetIsCompanionMode(b): + Compat._IsCompanionMode = b + @staticmethod + def SetIsBambu(b): + Compat._IsBambu = b _LocalAuthObj = None diff --git a/octoeverywhere/octopingpong.py b/octoeverywhere/octopingpong.py index d7de3ac..ca0c31f 100644 --- a/octoeverywhere/octopingpong.py +++ b/octoeverywhere/octopingpong.py @@ -212,8 +212,7 @@ def _ComputeStats(self, defaultServerResult): c += 1 # Keep track of which server we have the lowest result counts for. - if c < smallestBucketStatCount: - smallestBucketStatCount = c + smallestBucketStatCount = min(smallestBucketStatCount, c) # Prevent divide by zero if c == 0: diff --git a/py_installer/ConfigHelper.py b/py_installer/ConfigHelper.py new file mode 100644 index 0000000..4737c82 --- /dev/null +++ b/py_installer/ConfigHelper.py @@ -0,0 +1,174 @@ +import os + +from linux_host.config import Config + +from .Logging import Logger +from .Context import Context + +# Since the installer shares the common config class as the plugin, this helper helps the installer access it. +# Mostly, since the config is held in memory in the plugin, changes made by the installer should only hold the Config +# class for a short time and then flush it, and then ensure the plugin restarts shortly after it's touched by the installer. +class ConfigHelper: + + # + # Frontend + # + + # Given a context, this will try to find the config file and see if the frontend data is in it. + # If any data is found, it will be returned. If there's no config or the data doesn't exist, it will return None. + # Returns (portStr:str, frontendHint:str (can be None)) + @staticmethod + def TryToGetFrontendDetails(context:Context): + try: + # Load the config, if this returns None, there is no existing file. + c = ConfigHelper._GetConfig(context) + if c is None: + return (None, None) + # Use a default of None so if they don't exist, they aren't added to the config. + frontendPortStr = c.GetStr(Config.RelaySection, Config.RelayFrontEndPortKey, None) + frontendTypeHint = c.GetStr(Config.RelaySection, Config.RelayFrontEndTypeHintKey, None) + return (frontendPortStr, frontendTypeHint) + except Exception as e: + # There have been a few reports of this file being corrupt, so if it is, we will just fail and rewrite it. + Logger.Warn("Failed to parse frontend details from existing config. "+str(e)) + return (None, None) + + + # Writes the frontend details to the config file + @staticmethod + def WriteFrontendDetails(context:Context, portStr:str, frontendHint_CanBeNone:str): + try: + # Load the config, force it to be created if it doesn't exist. + c = ConfigHelper._GetConfig(context, createIfNotExisting=True) + # Write the new values + c.SetStr(Config.RelaySection, Config.RelayFrontEndPortKey, portStr) + c.SetStr(Config.RelaySection, Config.RelayFrontEndTypeHintKey, frontendHint_CanBeNone) + except Exception as e: + Logger.Error("Failed to write frontend details to config. "+str(e)) + raise Exception("Failed to write frontend details to config") from e + + + # + # Companion And Bambu Connect + # + + # Given a context, this will try to find the config file and see if the companion data is in it. + # These vars are shared for the companion and bambu connect logic. + # If any data is found, it will be returned. If there's no config or the data doesn't exist, it will return None. + # Returns (ipOrHostname:str, portStr:str) + @staticmethod + def TryToGetCompanionDetails(context:Context = None, configFolderPath:str = None): + try: + # Load the config, if this returns None, there is no existing file. + c = ConfigHelper._GetConfig(context, configFolderPath) + if c is None: + return (None, None) + # Use a default of None so if they don't exist, they aren't added to the config. + ipOrHostname = c.GetStr(Config.SectionCompanion, Config.CompanionKeyIpOrHostname, None) + portStr = c.GetStr(Config.SectionCompanion, Config.CompanionKeyPort, None) + return (ipOrHostname, portStr) + except Exception as e: + # There have been a few reports of this file being corrupt, so if it is, we will just fail and rewrite it. + Logger.Warn("Failed to parse companion details from existing config. "+str(e)) + return (None, None) + + + # Writes the companion details to the config file + @staticmethod + def WriteCompanionDetails(context:Context, ipOrHostname:str, portStr:str): + try: + # Load the config, force it to be created if it doesn't exist. + c = ConfigHelper._GetConfig(context, createIfNotExisting=True) + # Write the new values + c.SetStr(Config.SectionCompanion, Config.CompanionKeyIpOrHostname, ipOrHostname) + c.SetStr(Config.SectionCompanion, Config.CompanionKeyPort, portStr) + except Exception as e: + Logger.Error("Failed to write companion details to config. "+str(e)) + raise Exception("Failed to write companion details to config") from e + + + # + # Bambu Connect Only + # + + # Given a context, this will try to find the config file and see if the bambu data is in it. + # If any data is found, it will be returned. If there's no config or the data doesn't exist, it will return None. + # Returns (accessToken:str, printerSn:str) + @staticmethod + def TryToGetBambuData(context:Context = None, configFolderPath:str = None): + try: + # Load the config, if this returns None, there is no existing file. + c = ConfigHelper._GetConfig(context, configFolderPath) + if c is None: + return (None, None) + # Use a default of None so if they don't exist, they aren't added to the config. + accessToken = c.GetStr(Config.SectionBambu, Config.BambuAccessToken, None) + printerSn = c.GetStr(Config.SectionBambu, Config.BambuPrinterSn, None) + return (accessToken, printerSn) + except Exception as e: + # There have been a few reports of this file being corrupt, so if it is, we will just fail and rewrite it. + Logger.Warn("Failed to parse bambu details from existing config. "+str(e)) + return (None, None) + + + # Writes the bambu details to the config file + @staticmethod + def WriteBambuDetails(context:Context, accessToken:str, printerSn:str): + try: + # Load the config, force it to be created if it doesn't exist. + c = ConfigHelper._GetConfig(context, createIfNotExisting=True) + # Write the new values + c.SetStr(Config.SectionBambu, Config.BambuAccessToken, accessToken) + c.SetStr(Config.SectionBambu, Config.BambuPrinterSn, printerSn) + except Exception as e: + Logger.Error("Failed to write bambu details to config. "+str(e)) + raise Exception("Failed to write bambu details to config") from e + + + # + # Helpers + # + + # Given a context or folder path, this will return if there's any existing config file yet or not. + @staticmethod + def DoesConfigFileExist(context:Context = None, configFolderPath:str = None) -> bool: + configFilePath = None + if context is not None: + configFilePath = ConfigHelper.GetConfigFilePath(context) + elif configFolderPath is not None: + configFilePath = ConfigHelper.GetConfigFilePath(configFolderPath=configFolderPath) + else: + raise Exception("DoesConfigFileExist no context or file path passed.") + return os.path.exists(configFilePath) and os.path.isfile(configFilePath) + + + # Given a context or config file path, this returns file path of the config. + @staticmethod + def GetConfigFilePath(context:Context = None, configFolderPath:str = None): + if context is not None: + if context.ConfigFolder is None: + raise Exception("GetConfigFilePath context doesn't have a ConfigFolder string.") + return Config.GetConfigFilePath(context.ConfigFolder) + if configFolderPath is not None: + return Config.GetConfigFilePath(configFolderPath) + raise Exception("GetConfigFilePath was passed no config or config folder path.") + + + # Given a context or folder path, this returns a config object if it exists. + # If the file doesn't exist and createIfNotExisting is False, None is returned. + # Otherwise a new config will be created. + @staticmethod + def _GetConfig(context:Context = None, configFolderPath:str = None, createIfNotExisting:bool = False): + if ConfigHelper.DoesConfigFileExist(context, configFolderPath) is False: + if createIfNotExisting: + # Fallthrough, the Config class will create a file if none exists. + Logger.Info("Creating main plugin config file.") + else: + return None + # Get the config folder path. + if configFolderPath is None: + configFolderPath = context.ConfigFolder + if configFolderPath is None: + raise Exception("_GetConfig was called with an invalid context and now config folder path.") + # Open or create the config. + return Config(configFolderPath) diff --git a/py_installer/Configure.py b/py_installer/Configure.py new file mode 100644 index 0000000..c744f1c --- /dev/null +++ b/py_installer/Configure.py @@ -0,0 +1,140 @@ +import os + +from .Util import Util +from .Paths import Paths +from .Logging import Logger +from .Context import Context +from .Context import OsTypes +from .NetworkConnectors.MoonrakerConnector import MoonrakerConnector +from .NetworkConnectors.BambuConnector import BambuConnector + +# The goal of this class is the take the context object from the Discovery Gen2 phase to the Phase 3. +class Configure: + + # This is the common service prefix (or word used in the file name) we use for all of our service file names. + # This MUST be used for all instances running on this device, both local plugins and companions. + # This also MUST NOT CHANGE, as it's used by the Updater logic to find all of the locally running services. + c_ServiceCommonName = "octoeverywhere" + + def Run(self, context:Context): + + Logger.Header("Starting configuration...") + + # Figure the service suffix. + # All services start with octoeverywhere (c_ServiceCommonName) but they have different suffixes so they don't collide if there + # are multiple instances or types install one a signal device. + serviceSuffixStr = "" + if context.IsCompanionOrBambu(): + # For companions or bambu, we use the companion id, with a unique prefix to separate it from any possible local installs + # Special case, for the primary instance id, we don't add the number suffix, so it easier to use. + instanceIdSuffix = "" if context.IsPrimaryCompanionOrBambu() else f"-{context.CompanionInstanceId}" + pluginTypeStr = "bambu" if context.IsBambuSetup else "companion" + serviceSuffixStr = f"-{pluginTypeStr}{instanceIdSuffix}" + elif context.OsType == OsTypes.SonicPad: + # For Sonic Pad, we know the format of the service file is a bit different. + # For the SonicIt is moonraker_service or moonraker_service. + if "." in context.MoonrakerServiceFileName: + serviceSuffixStr = context.MoonrakerServiceFileName.split(".")[1] + elif context.OsType == OsTypes.K1: + # For the k1, there's only every one moonraker instance, so this isn't needed. + pass + else: + # Now we need to figure out the instance suffix we need to use. + # To keep with Kiauh style installs, each moonraker instances will be named moonraker-.service. + # If there is only one moonraker instance, the name is moonraker.service. + # Default to empty string, which means there's no suffix and only one instance. + serviceFileNameNoExtension = context.MoonrakerServiceFileName.split('.')[0] + if '-' in serviceFileNameNoExtension: + moonrakerServiceSuffix = serviceFileNameNoExtension.split('-') + serviceSuffixStr = "-" + moonrakerServiceSuffix[1] + Logger.Debug(f"Moonraker Service File Name: {context.MoonrakerServiceFileName}, Suffix: '{serviceSuffixStr}'") + + if context.IsCompanionOrBambu(): + # For companion or bambu setups, there is no local moonraker config files, so things are setup differently. + # The plugin data folder, which is normally the root printer data folder for that instance, becomes our per instance companion folder. + context.RootFolder = context.CompanionDataRoot + # The config folder is where our config lives, which we put in the main data root. + context.ConfigFolder = context.RootFolder + # Make a logs folder, so it's found bellow + Util.EnsureDirExists(os.path.join(context.RootFolder, "logs"), context, True) + elif context.OsType == OsTypes.SonicPad: + # ONLY FOR THE SONIC PAD, we know the folder setup is different. + # The user data folder will have /mnt/UDISK/printer_config where the config files are and /mnt/UDISK/printer_logs for logs. + # Use the normal folder for the config files. + context.ConfigFolder = Util.GetParentDirectory(context.MoonrakerConfigFilePath) + + # There really is no printer data folder, so make one that's unique per instance. + # So based on the config folder, go to the root of it, and them make the folder "octoeverywhere_data" + context.RootFolder = os.path.join(Util.GetParentDirectory(context.ConfigFolder), f"octoeverywhere_data{serviceSuffixStr}") + Util.EnsureDirExists(context.RootFolder, context, True) + else: + # For now we assume the folder structure is the standard Klipper folder config, + # thus the full moonraker config path will be .../something_data/config/moonraker.conf + # Based on that, we will define the config folder and the printer data root folder. + # Note that the K1 uses this standard folder layout as well. + context.ConfigFolder = Util.GetParentDirectory(context.MoonrakerConfigFilePath) + context.RootFolder = Util.GetParentDirectory(context.ConfigFolder) + Logger.Debug("Printer data folder: "+context.RootFolder) + + + # This is the name of our service we create. If the port is the default port, use the default name. + # Otherwise, add the port to keep services unique. + if context.OsType == OsTypes.SonicPad: + # For Sonic Pad, since the service is setup differently, follow the conventions of it. + # Both the service name and the service file name must match. + # The format is _service + # NOTE! For the Update class to work, the name must start with Configure.c_ServiceCommonNamePrefix + context.ServiceName = Configure.c_ServiceCommonName + "_service" + if len(serviceSuffixStr) != 0: + context.ServiceName= context.ServiceName + "." + serviceSuffixStr + context.ServiceFilePath = os.path.join(Paths.CrealityOsServiceFilePath, context.ServiceName) + elif context.OsType == OsTypes.K1: + # For the k1, there's only ever one moonraker and we know the exact service naming convention. + # Note we use 66 to ensure we start after moonraker. + # This is page for details on the file name: https://docs.oracle.com/cd/E36784_01/html/E36882/init.d-4.html + # Note the 'S66' string is looked for in the plugin's EnsureUpdateManagerFilesSetup function. So it must not change! + context.ServiceName = f"S66{Configure.c_ServiceCommonName}_service" + context.ServiceFilePath = os.path.join(Paths.CrealityOsServiceFilePath, context.ServiceName) + else: + # For normal setups, use the convention that Klipper users + # NOTE! For the Update class to work, the name must start with Configure.c_ServiceCommonNamePrefix + context.ServiceName = Configure.c_ServiceCommonName + serviceSuffixStr + context.ServiceFilePath = os.path.join(Paths.SystemdServiceFilePath, context.ServiceName+".service") + + # Since the moonraker config folder is unique to the moonraker instance, we will put our storage in it. + # This also prevents the user from messing with it accidentally. + context.LocalFileStorageFolder = os.path.join(context.RootFolder, "octoeverywhere-store") + + # Ensure the storage folder exists and is owned by the correct user. + Util.EnsureDirExists(context.LocalFileStorageFolder, context, True) + + # There's not a great way to find the log path from the config file, since the only place it's located is in the systemd file. + context.LogsFolder = None + + # First, we will see if we can find a named folder relative to this folder. + # This is the folder created for companion and bambu setups, so it should always exist in those cases. + context.LogsFolder = os.path.join(context.RootFolder, "logs") + if os.path.exists(context.LogsFolder) is False: + # Try an older path + context.LogsFolder = os.path.join(context.RootFolder, "klipper_logs") + if os.path.exists(context.LogsFolder) is False: + # Try the path Creality OS uses, something like /mnt/UDISK/printer_logs + context.LogsFolder = os.path.join(Util.GetParentDirectory(context.PrinterDataConfigFolder), f"printer_logs{serviceSuffixStr}") + if os.path.exists(context.LogsFolder) is False: + # Failed, make a folder in the printer data root. + context.LogsFolder = os.path.join(context.RootFolder, "octoeverywhere-logs") + # Create the folder and force the permissions so our service can write to it. + Util.EnsureDirExists(context.LogsFolder, context, True) + + # If this is an companion setup we need to setup the connection with moonraker and save it into the config. + if context.IsCompanionSetup: + mc = MoonrakerConnector() + mc.EnsureCompanionMoonrakerConnection(context) + + # If this is a Bambu Connect setup, we need to make sure we have a connection to a Bambu printer. + if context.IsBambuSetup: + bc = BambuConnector() + bc.EnsureBambuConnection(context) + + # Report + Logger.Info(f'Configured. Service: {context.ServiceName}, Path: {context.ServiceFilePath}, LocalStorage: {context.LocalFileStorageFolder}, Config Dir: {context.ConfigFolder}, Logs: {context.LogsFolder}') diff --git a/moonraker_installer/Context.py b/py_installer/Context.py similarity index 77% rename from moonraker_installer/Context.py rename to py_installer/Context.py index e9674c7..0e8fb47 100644 --- a/moonraker_installer/Context.py +++ b/py_installer/Context.py @@ -21,6 +21,11 @@ class OsTypes(Enum): # Generation 3 - Must exist after the configure phase. class Context: + # For companions or bambu connect plugins, the primary instance is a little special. + # It will have an instance ID of 1, but when we use the id we want to exclude the suffix. + # This is so the first instance will have a normal name like "octoeverywhere-bambu" instead of "octoeverywhere-bambu-1" + CompanionPrimaryInstanceId = "1" + def __init__(self) -> None: # @@ -59,8 +64,11 @@ def __init__(self) -> None: # Parsed from the command line args, if set, we shouldn't auto select the moonraker instance. self.DisableAutoMoonrakerInstanceSelection:bool = False - # Parsed from the command line args, if set, this plugin should be installed as an observer. - self.IsObserverSetup:bool = False + # Parsed from the command line args, if set, this plugin should be installed as an companion. + self.IsCompanionSetup:bool = False + + # Parsed from the command line args, if set, this plugin should be installed as an bambu connect (similar to the companion). + self.IsBambuSetup:bool = False # Parsed from the command line args, if set, the plugin install should be in update mode. self.IsUpdateMode:bool = False @@ -80,40 +88,41 @@ def __init__(self) -> None: self.MoonrakerServiceFileName:str = None ### - OR - ### - # These values will be filled out if this is an observer setup. + # These values will be filled out if this is a companion OR Bambu connect setup. - # The root folder where this plugin instance will setup it's data. - self.ObserverDataPath:str = None + # The root folder where the companion or Bambu plugin data lives. + self.CompanionDataRoot:str = None - # The observer instance id, so we can support multiple instances on one device. - self.ObserverInstanceId:str = None + # The companion or bambu instance id, so we can support multiple instances on one device. + # Note that a id of "1" is special, and you can use IsPrimaryCompanionOrBambu to detect it. + self.CompanionInstanceId:str = None # # Generation 3 # - # Generation 3 - This it the path to the printer data root folder. - self.PrinterDataFolder:str = None + # For local plugin configs, this is the printer data folder root. + # For companion or bambu plugins, this is the same as self.CompanionDataRoot + self.RootFolder:str = None + + # This the folder where our main plugin config is or will be. + # For local plugin configs, this is the Moonraker config folder. + # For companion or bambu plugins, this is the same as self.CompanionDataRoot + self.ConfigFolder:str = None - # Generation 3 - This it the path to the printer data config folder. - self.PrinterDataConfigFolder:str = None + # This is the folder where the plugin logs will go. + self.LogsFolder:str = None - # Generation 3 - This it the path to the printer data logs folder. - self.PrinterDataLogsFolder:str = None + # The path to where the local storage will be put for this instance. + self.LocalFileStorageFolder:str = None - # Generation 3 - This is the name of this OctoEverywhere instance's service. + # This is the name of this OctoEverywhere instance's service. self.ServiceName:str = None - # Generation 3 - The full file path and file name of this instance's service file. + # The full file path and file name of this instance's service file. self.ServiceFilePath:str = None - # Generation 3 - The path to where the local storage will be put for this instance. - self.LocalFileStorageFolder:str = None - - # Generation 3 - Only set if this is an observer setup - self.ObserverConfigFilePath:str = None - # # Generation 4 # @@ -127,6 +136,18 @@ def IsCrealityOs(self) -> bool: return self.OsType == OsTypes.SonicPad or self.OsType == OsTypes.K1 + # Returns true if the target is a companion or bambu connect setup. + def IsCompanionOrBambu(self) -> bool: + return self.IsCompanionSetup or self.IsBambuSetup + + + # Returns true if this is a bambu or companion plugin and it's the primary, aka it has an instance ID of 1. + def IsPrimaryCompanionOrBambu(self) -> bool: + if self.IsCompanionOrBambu() is False: + raise Exception("IsPrimaryCompanionOrBambu was called for a non companion or bambu context.") + return self.CompanionInstanceId == Context.CompanionPrimaryInstanceId + + @staticmethod def LoadFromArgString(argString:str): Logger.Debug("Found config: "+argString) @@ -161,11 +182,11 @@ def Validate(self, generation = 1) -> None: self.CmdLineArgs = self.CmdLineArgs.strip() if generation >= 2: - if self.IsObserverSetup: - self._ValidatePathAndExists(self.ObserverDataPath, "Required config var Observer Data Path was not found") - self._ValidateString(self.ObserverInstanceId, "Required config var Observer Instance Id was not found") - self.ObserverDataPath = self.ObserverDataPath.strip() - self.ObserverInstanceId = self.ObserverInstanceId.strip() + if self.IsCompanionOrBambu(): + self._ValidatePathAndExists(self.CompanionDataRoot, "Required config var Companion Data Path was not found") + self._ValidateString(self.CompanionInstanceId, "Required config var Companion Instance Id was not found") + self.CompanionDataRoot = self.CompanionDataRoot.strip() + self.CompanionInstanceId = self.CompanionInstanceId.strip() if self.OsType != OsTypes.Debian: raise Exception("The OctoEverywhere companion can only be installed on Debian based operating systems.") else: @@ -175,16 +196,13 @@ def Validate(self, generation = 1) -> None: self.MoonrakerServiceFileName = self.MoonrakerServiceFileName.strip() if generation >= 3: - self._ValidatePathAndExists(self.PrinterDataFolder, "Required config var Printer Data Folder was not found") - self._ValidatePathAndExists(self.PrinterDataConfigFolder, "Required config var Printer Data Config Folder was not found") - self._ValidatePathAndExists(self.PrinterDataLogsFolder, "Required config var Printer Data Logs Folder was not found") - self._ValidatePathAndExists(self.PrinterDataLogsFolder, "Required config var Printer Data Logs Folder was not found") + self._ValidatePathAndExists(self.RootFolder, "Required config var Root Folder was not found") + self._ValidatePathAndExists(self.ConfigFolder, "Required config var Config Folder was not found") + self._ValidatePathAndExists(self.LogsFolder, "Required config var Logs Folder was not found") self._ValidatePathAndExists(self.LocalFileStorageFolder, "Required config var local storage folder was not found") # This path wont exist on the first install, because it won't be created until the end of the install. self._ValidateString(self.ServiceFilePath, "Required config var service file path was not found") self._ValidateString(self.ServiceName, "Required config var service name was not found") - if self.IsObserverSetup: - self._ValidatePathAndExists(self.ObserverConfigFilePath, "Required config var Observer Config File Path was not found") if generation >= 4: # The printer ID can be None, this means it didn't exist before we installed the service. @@ -225,10 +243,13 @@ def ParseCmdLineArgs(self): elif rawArg.lower() == "observer": # This is the legacy flag Logger.Info("Setup running in companion setup mode.") - self.IsObserverSetup = True + self.IsCompanionSetup = True elif rawArg.lower() == "companion": Logger.Info("Setup running in companion setup mode.") - self.IsObserverSetup = True + self.IsCompanionSetup = True + elif rawArg.lower() == "bambu": + Logger.Info("Setup running in Bambu Connect setup mode.") + self.IsBambuSetup = True elif rawArg.lower() == "update": Logger.Info("Setup running in update mode.") self.IsUpdateMode = True diff --git a/moonraker_installer/Discovery.py b/py_installer/Discovery.py similarity index 99% rename from moonraker_installer/Discovery.py rename to py_installer/Discovery.py index 99025d2..ae75054 100644 --- a/moonraker_installer/Discovery.py +++ b/py_installer/Discovery.py @@ -96,7 +96,7 @@ def FindTargetMoonrakerFiles(self, context:Context): isFirstPrint = False else: Logger.Warn( "If you need help, contact us! https://octoeverywhere.com/support") - response = input("Enter the number for the config you would like to setup now: ") + response = input("Enter the number from above for the Moonraker you would like to setup now: ") response = response.lower().strip() # Parse the input and -1 it, so it aligns with the array length. tempInt = int(response.lower().strip()) - 1 diff --git a/py_installer/DiscoveryCompanionAndBambu.py b/py_installer/DiscoveryCompanionAndBambu.py new file mode 100644 index 0000000..5a6b692 --- /dev/null +++ b/py_installer/DiscoveryCompanionAndBambu.py @@ -0,0 +1,143 @@ +import os + +from .Logging import Logger +from .Context import Context +from .Util import Util +from .ConfigHelper import ConfigHelper + +# This class does the same function as the Discovery class, but for companion or Bambu Connect plugins. +# Note that "Bambu Connect" is really just a type of companion plugin, but we use different names so it feels correct. +class DiscoveryCompanionAndBambu: + + # This is the base data folder name that will be used, the plugin id suffix will be added to end of it. + # The folders will always be in the user's home path. + # These MUST start with a . and be LOWER CASE for the matching logic below to work correctly! + # The primary instance (id == "1") will have no "-#" suffix on the folder or service name. + c_CompanionPluginDataRootFolder_Lower = ".octoeverywhere-companion" + c_BambuPluginDataRootFolder_Lower = ".octoeverywhere-bambu" + + + def Discovery(self, context:Context): + Logger.Debug("Starting companion discovery.") + + # Used for printing the type, like "would you like to install a new {pluginTypeStr} plugin?" + pluginTypeStr = "Bambu Connect" if context.IsBambuSetup else "Companion" + + # Look for existing companion or bambu data installs. + existingCompanionFolders = [] + # Sort so the folder we find are ordered from 1-... This makes the selection process nicer, since the ID == selection. + fileAndDirList = sorted(os.listdir(context.UserHomePath)) + for fileOrDirName in fileAndDirList: + # Use starts with to see if it matches any of our possible folder names. + # Since each setup only targets companion or bambu connect, only pick the right folder type. + fileOrDirNameLower = fileOrDirName.lower() + if context.IsCompanionSetup: + if fileOrDirNameLower.startswith(DiscoveryCompanionAndBambu.c_CompanionPluginDataRootFolder_Lower): + existingCompanionFolders.append(fileOrDirName) + Logger.Debug(f"Found existing companion data folder: {fileOrDirName}") + elif context.IsBambuSetup: + if fileOrDirNameLower.startswith(DiscoveryCompanionAndBambu.c_BambuPluginDataRootFolder_Lower): + existingCompanionFolders.append(fileOrDirName) + Logger.Debug(f"Found existing bambu data folder: {fileOrDirName}") + else: + raise Exception("DiscoveryCompanionAndBambu used in non companion or bambu connect context.") + + # If there's an existing folders, ask the user if they want to use them. + if len(existingCompanionFolders) > 0: + count = 1 + Logger.Blank() + Logger.Header(f"Existing {pluginTypeStr} Plugins Found") + Logger.Blank() + Logger.Info( "If you want to update or recover an existing plugin enter the Plugin ID from the list below.") + Logger.Info( " - or - ") + Logger.Info(f"If you want to install a new {pluginTypeStr} plugin, enter 'n'.") + Logger.Blank() + Logger.Info("Options:") + for folder in existingCompanionFolders: + instanceId = self._GetCompanionOrBambuIdFromFolderName(folder) + # Try to parse the config, if there is one and it's valid. + ip, port = ConfigHelper.TryToGetCompanionDetails(configFolderPath=os.path.join(context.UserHomePath, folder)) + if ip is None and port is None: + Logger.Info(f" {count}) Plugin ID {instanceId} - Path: {folder}") + else: + Logger.Info(f" {count}) Plugin ID {instanceId} - {ip}:{port}") + count += 1 + Logger.Info(f" n) Setup a new {pluginTypeStr} plugin instance") + Logger.Blank() + # Ask the user which number they want. + responseInt = -1 + isFirstPrint = True + while True: + try: + if isFirstPrint: + isFirstPrint = False + else: + Logger.Warn( "If you need help, contact us! https://octoeverywhere.com/support") + response = input("Enter a Plugin ID from the list above or 'n': ") + response = response.lower().strip() + # If the response is n, fall through. + if response == "n": + break + # Parse the input and -1 it, so it aligns with the array length. + tempInt = int(response.lower().strip()) - 1 + if tempInt >= 0 and tempInt < len(existingCompanionFolders): + responseInt = tempInt + break + Logger.Blank() + Logger.Warn("Invalid number selection, try again.") + except Exception as _: + Logger.Blank() + Logger.Warn("Invalid input, try again.") + + # If there is a response, the user selected an instance. + if responseInt != -1: + # Use this instance + self._SetupContextFromVars(context, existingCompanionFolders[responseInt]) + Logger.Info(f"Existing {pluginTypeStr} plugin selected. Path: {context.CompanionDataRoot}, Id: {context.CompanionInstanceId}") + return + + # Create a new instance path. Either there is no existing data path or the user wanted to create a new one. + # There is a special case for instance ID "1", we use no suffix. All others will have the suffix. + newId = str(len(existingCompanionFolders) + 1) + folderNameRoot = DiscoveryCompanionAndBambu.c_BambuPluginDataRootFolder_Lower if context.IsBambuSetup else DiscoveryCompanionAndBambu.c_CompanionPluginDataRootFolder_Lower + fullFolderName = folderNameRoot if newId == Context.CompanionPrimaryInstanceId else f"{folderNameRoot}-{newId}" + self._SetupContextFromVars(context, fullFolderName) + Logger.Info(f"Creating a new {pluginTypeStr} plugin data path. Path: {context.CompanionDataRoot}, Id: {context.CompanionInstanceId}") + return + + + def _SetupContextFromVars(self, context:Context, folderName:str): + # First, ensure we can parse the id and set it. + context.CompanionInstanceId = self._GetCompanionOrBambuIdFromFolderName(folderName) + + # Make the full path + context.CompanionDataRoot = os.path.join(context.UserHomePath, folderName) + + # Ensure the file exists and we have permissions + Util.EnsureDirExists(context.CompanionDataRoot, context, True) + + + # Returns the instance id, for primary instances, this returns "1" + def _GetCompanionOrBambuIdFromFolderName(self, folderName:str): + folderName_lower = folderName.lower() + + # If the folder name starts with any of these, then its a folder we can get the instance for. + # We will get the suffix for the folder path and then figure out the id. + folderSuffix = None + if folderName_lower.startswith(DiscoveryCompanionAndBambu.c_CompanionPluginDataRootFolder_Lower) is True: + folderSuffix = folderName_lower[len(DiscoveryCompanionAndBambu.c_CompanionPluginDataRootFolder_Lower):] + elif folderName_lower.startswith(DiscoveryCompanionAndBambu.c_BambuPluginDataRootFolder_Lower) is True: + folderSuffix = folderName_lower[len(DiscoveryCompanionAndBambu.c_BambuPluginDataRootFolder_Lower):] + else: + Logger.Error(f"We tried to get an companion or bambu connect ID from a non-companion or bambu connect data folder. {folderName}") + raise Exception("We tried to get an companion or bambu connect ID from a non-companion or bambu connect data folder") + + # If there is no suffix, this is the primary instance + if folderSuffix is None or len(folderSuffix) == 0: + return Context.CompanionPrimaryInstanceId + + # Otherwise, remove the - and return the id. + if folderSuffix.startswith("-") is False: + Logger.Error(f"We tried to get an companion or bambu connect ID but the suffix didn't start with a -. {folderName}") + raise Exception("We tried to get an companion or bambu connect ID but the suffix didn't start with a -") + return folderSuffix[1:] diff --git a/moonraker_installer/Frontend.py b/py_installer/Frontend.py similarity index 76% rename from moonraker_installer/Frontend.py rename to py_installer/Frontend.py index 1253021..ebaa2b8 100644 --- a/moonraker_installer/Frontend.py +++ b/py_installer/Frontend.py @@ -1,14 +1,10 @@ -import os from enum import Enum -import configparser import requests -from moonraker_octoeverywhere.config import Config - +from .Util import Util from .Logging import Logger from .Context import Context -from .ObserverConfigFile import ObserverConfigFile -from .Util import Util +from .ConfigHelper import ConfigHelper # Frontends that are known. class KnownFrontends(Enum): @@ -34,10 +30,16 @@ class Frontend: # If called, this will walk the user though picking a frontend for the targeted device (local or remote for companion). # The frontend will be saved into the OE config, so this should be done before the service starts, if this is a first time run. def DoFrontendSetup(self, context:Context): + + # There's no frontend for bambu connect. + if context.IsBambuSetup: + Logger.Debug("Skipping frontend setup, there's no frontend for bambu connect.") + return + Logger.Header("Starting Web Interface Setup") # Try to get the existing configured port. - (currentPort, frontendHint_CanBeNone) = self._TryToReadCurrentFrontendSetup(context) + (currentPort, frontendHint_CanBeNone) = ConfigHelper.TryToGetFrontendDetails(context) if currentPort is not None: # There is already a config with a port setup. # Ask if the user wants to keep the current setup. @@ -59,7 +61,7 @@ def DoFrontendSetup(self, context:Context): (portInt, frontendHint_CanBeNone) = self._GetDesiredFrontend(context) # We got a port, save it. - self._WriteFrontendSetup(context, str(portInt), frontendHint_CanBeNone) + ConfigHelper.WriteFrontendDetails(context, str(portInt), frontendHint_CanBeNone) # Returns the (port (int), frontendNameHint:str or None) of the frontend the user wants to use. @@ -67,8 +69,8 @@ def _GetDesiredFrontend(self, context:Context): # Find the target. If this is a local install, the target is local. # Otherwise, it's whatever the companion target is. targetIpOrHostname = "127.0.0.1" - if context.IsObserverSetup: - (ip, _) = ObserverConfigFile.TryToParseConfig(context.ObserverConfigFilePath) + if context.IsCompanionOrBambu(): + (ip, _) = ConfigHelper.TryToGetCompanionDetails(context) if ip is None or len(ip) == 0: raise Exception("Frontend setup failed to find companion ip from companion config file.") targetIpOrHostname = ip @@ -267,66 +269,3 @@ def CheckIfValidFrontend(self, ipOrHostname:str, portStr:str, timeoutSec:float = Logger.Debug(f"Frontend check failed. {ipOrHostname}:{portStr} {str(e)}") # Return failed. return (False, False, KnownFrontends.Unknown) - - - # Returns the current configured port and frontend name hint. - # (portStr:str, frontendHint:str (can be None)) - # Returns (None, None) if the file can't be found. - def _TryToReadCurrentFrontendSetup(self, context:Context): - filePath = self.GetOctoEverywhereServiceConfigFilePath(context) - # Don't try catch, let this throw if there's a problem reading the config, - # That would be bad. - if os.path.exists(filePath) is False: - return (None, None) - - try: - config = configparser.ConfigParser() - config.read(filePath) - if config.has_section(Config.RelaySection) is False: - return (None, None) - if Config.RelayFrontEndPortKey not in config[Config.RelaySection]: - return (None, None) - portStr = config[Config.RelaySection][Config.RelayFrontEndPortKey] - frontendHint = None - if Config.RelayFrontEndTypeHintKey in config[Config.RelaySection]: - frontendHint = config[Config.RelaySection][Config.RelayFrontEndTypeHintKey] - return (portStr, frontendHint) - except Exception as e: - # There have been a few reports of this file being corrupt, so if it is, we will just fail and rewrite it. - Logger.Warn(f"Failed to read the frontend setup from the config file. {str(e)}") - return (None, None) - - - # Writes the current frontend setup into the main OE service config - # We use the main config file, since it's already there, and we don't want to have overlapping settings in different places. - # If the service hasn't ran yet, the file won't exist, in which case we will create it. - def _WriteFrontendSetup(self, context:Context, portStr:str, frontendHint_CanBeNone:str): - filePath = self.GetOctoEverywhereServiceConfigFilePath(context) - - # Read the current config if there is one, this is important. - config = configparser.ConfigParser() - if os.path.exists(filePath): - config.read(filePath) - # Add the vars - if config.has_section(Config.RelaySection) is False: - config.add_section(Config.RelaySection) - config[Config.RelaySection][Config.RelayFrontEndPortKey] = portStr - if frontendHint_CanBeNone is None: - if Config.RelayFrontEndTypeHintKey in config[Config.RelaySection]: - del config[Config.RelaySection][Config.RelayFrontEndTypeHintKey] - else: - config[Config.RelaySection][Config.RelayFrontEndTypeHintKey] = frontendHint_CanBeNone - # Write the file back or make a new one. - with open(filePath, 'w', encoding="utf-8") as f: - config.write(f) - - # Important! If we were the first ones to create this file, it will be owned by root and the service - # won't have permission to open it. Thus we need to make sure it's owned correctly when we are done. - Util.SetFileOwnerRecursive(filePath, context.UserName) - - - def GetOctoEverywhereServiceConfigFilePath(self, context:Context) -> str: - # Don't do the join if there is no path, otherwise the result will just be a file name. - if context.PrinterDataConfigFolder is None or len(context.PrinterDataConfigFolder) == 0: - return None - return os.path.join(context.PrinterDataConfigFolder, Config.ConfigFileName) diff --git a/moonraker_installer/Installer.py b/py_installer/Installer.py similarity index 81% rename from moonraker_installer/Installer.py rename to py_installer/Installer.py index 1d97940..d0bd609 100644 --- a/moonraker_installer/Installer.py +++ b/py_installer/Installer.py @@ -6,7 +6,7 @@ from .Service import Service from .Context import Context, OsTypes from .Discovery import Discovery -from .DiscoveryObserver import DiscoveryObserver +from .DiscoveryCompanionAndBambu import DiscoveryCompanionAndBambu from .Configure import Configure from .Updater import Updater from .Permissions import Permissions @@ -89,7 +89,7 @@ def _RunInternal(self): # Ensure that the system clock sync is enabled. For some MKS PI systems the OS time is wrong and sync is disabled. # The user would of had to manually correct the time to get this installer running, but we will ensure that the # time sync systemd service is enabled to keep the clock in sync after reboots, otherwise it will cause SSL errors. - TimeSync.EnsureNtpSyncEnabled() + TimeSync.EnsureNtpSyncEnabled(context) # Ensure the script at least has sudo permissions. # It's required to set file permission and to write / restart the service. @@ -98,8 +98,7 @@ def _RunInternal(self): # If we are in update mode, do the update logic and exit. if context.IsUpdateMode: - # Before the update, make sure all permissions are set - # correctly. + # Before the update, make sure all permissions are set correctly. permissions.EnsureFinalPermissions(context) # Do the update logic. @@ -114,11 +113,11 @@ def _RunInternal(self): return # Next step is to discover and fill out the moonraker config file path and service file name. - # If we are doing an observer setup, we need the user to help us input the details to the external moonraker IP. + # If we are doing an companion or bambu setup, we need the user to help us input the details to the external moonraker IP or bambu printer. # This is the hardest part of the setup, because it's highly dependent on the system and different moonraker setups. - if context.IsObserverSetup: - discovery = DiscoveryObserver() - discovery.ObserverDiscovery(context) + if context.IsCompanionOrBambu(): + discovery = DiscoveryCompanionAndBambu() + discovery.Discovery(context) else: discovery = Discovery() discovery.FindTargetMoonrakerFiles(context) @@ -156,8 +155,8 @@ def _RunInternal(self): # Add our auto update logic. updater = Updater() - # If this is an observer or a Creality OS install, put the update script in the users root, so it's easy to find. - if context.IsObserverSetup or context.IsCrealityOs(): + # If this is an companion or a Creality OS install, put the update script in the users root, so it's easy to find. + if context.IsCompanionOrBambu() or context.IsCrealityOs(): updater.PlaceUpdateScriptInRoot(context) # Also setup our cron updater if we can, so that the plugin will auto update. updater.EnsureCronUpdateJob(context.RepoRootFolder) @@ -170,7 +169,7 @@ def _RunInternal(self): Logger.Blank() Logger.Blank() Logger.Blank() - Logger.Purple(" ~~~ OctoEverywhere For Klipper Setup Complete ~~~ ") + Logger.Purple(" ~~~ OctoEverywhere Setup Complete ~~~ ") Logger.Warn( " You Can Access Your Printer Anytime From OctoEverywhere.com") Logger.Header(" Welcome To Our Community ") Logger.Error( " <3 ") @@ -204,12 +203,18 @@ def PrintHelp(self): Logger.Blank() Logger.Blank() Logger.Blank() - Logger.Header("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") - Logger.Header(" OctoEverywhere For Klipper ") - Logger.Header("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + Logger.Header("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + Logger.Header(" OctoEverywhere For Klipper And Bambu Connect ") + Logger.Header("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") Logger.Blank() - Logger.Info("This installer script is used for installing the OctoEverywhere plugin on Klipper/Moonraker/Mainsail/Fluidd setups. It is NOT used for OctoPrint setups.") - Logger.Info("If you want to install OctoEverywhere for OctoPrint, use the plugin manager in OctoPrint's settings to install the plugin.") + Logger.Info("This installer can be used for:") + Logger.Info(" - OctoEverywhere for Klipper - Where Moonraker is running on this device.") + Logger.Info(" - OctoEverywhere for Creality - Where this device is a Creality device (Sonic Pad, K1, Ender v3, etc)") + Logger.Info(" - OctoEverywhere Companion - Where this plugin will connect to Moonraker running on a different device on the same LAN.") + Logger.Info(" - OctoEverywhere Bambu Connect - Where this plugin will connect to a Bambu Labs printer on the same LAN.") + Logger.Blank() + Logger.Warn("This installer is NOT for:") + Logger.Info(" - OctoPrint or OctoKlipper - If you're using OctoPrint, install OctoEverywhere directly in OctoPrint from the plugin manager.") Logger.Blank() Logger.Warn("Command line format:") Logger.Info(" -other -args") @@ -218,10 +223,12 @@ def PrintHelp(self): Logger.Info(" - optional - If supplied, the install will target this moonraker setup without asking or searching for others") Logger.Info(" - optional - If supplied, the install will target this moonraker service file without searching.") Logger.Info(" Used when multiple moonraker instances are ran on the same device. The service name is used to find the unique moonraker identifier. OctoEverywhere will follow the same naming convention. Typically the file name is something like `moonraker-1.service` or `moonraker-somename.service`") - Logger.Info(" -observer - optional flag - If passed, the plugin is setup as an observer, which is a plugin not running on the same device as moonraker. This is useful for built-in printer hardware where OctoEverywhere can't run, like the Sonic Pad or K1.") Logger.Blank() Logger.Warn("Other Optional Args:") Logger.Info(" -help - Shows this message.") + Logger.Info(" -update - The installer will update all OctoEverywhere plugins on this device of any type.") + Logger.Info(" -companion - Makes the setup target a OctoEverywhere Companion plugin setup.") + Logger.Info(" -bambu - Makes the setup target a OctoEverywhere Bambu Connect plugin setup.") Logger.Info(" -noatuoselect - Disables auto selecting a moonraker instance, allowing the user to always choose.") Logger.Info(" -debug - Enable debug logging to the console.") Logger.Info(" -skipsudoactions - Skips sudo required actions. This is useful for debugging, but will make the install not fully work.") diff --git a/moonraker_installer/Linker.py b/py_installer/Linker.py similarity index 98% rename from moonraker_installer/Linker.py rename to py_installer/Linker.py index f1aa8e0..41ccff4 100644 --- a/moonraker_installer/Linker.py +++ b/py_installer/Linker.py @@ -38,7 +38,7 @@ def Run(self, context:Context): Logger.Blank() Logger.Blank() Logger.Error("We didn't get a response from the OctoEverywhere service when waiting for the printer id.") - Logger.Error("You can find service logs which might indicate the error in: "+context.PrinterDataLogsFolder) + Logger.Error("You can find service logs which might indicate the error in: "+context.LogsFolder) Logger.Blank() Logger.Blank() Logger.Error("Attempting to print the service logs:") @@ -98,7 +98,7 @@ def Run(self, context:Context): Logger.Blank() Logger.Blank() Logger.Error("The plugin hasn't connected to our service yet. Something might be wrong.") - Logger.Error("You can find service logs which might indicate the Logger.Error in: "+context.PrinterDataLogsFolder) + Logger.Error("You can find service logs which might indicate the Logger.Error in: "+context.LogsFolder) Logger.Blank() Logger.Blank() Logger.Error("Attempting to print the service logs:") diff --git a/moonraker_installer/Logging.py b/py_installer/Logging.py similarity index 100% rename from moonraker_installer/Logging.py rename to py_installer/Logging.py diff --git a/py_installer/NetworkConnectors/BambuConnector.py b/py_installer/NetworkConnectors/BambuConnector.py new file mode 100644 index 0000000..a81eeb1 --- /dev/null +++ b/py_installer/NetworkConnectors/BambuConnector.py @@ -0,0 +1,259 @@ +import threading +import socket + +from octoeverywhere.websocketimpl import Client + +from py_installer.Util import Util +from py_installer.Logging import Logger +from py_installer.Context import Context +from py_installer.ConfigHelper import ConfigHelper + + +# A class that helps the user discover, connect, and setup the details required to connect to a remote Bambu Labs printer. +class BambuConnector: + + def EnsureBambuConnection(self, context:Context): + Logger.Debug("Running bambu connect ensure config logic.") + + # For Bambu printers, we need the IP or Hostname, the port is static, + # and we also need the printer SN and access token. + ip, port = ConfigHelper.TryToGetCompanionDetails(context) + accessToken, printerSn = ConfigHelper.TryToGetBambuData(context) + if ip is not None and port is not None and accessToken is not None and printerSn is not None: + # Check if we can still connect. This can happen if the IP address changes, the user might need to setup the printer again. + Logger.Info(f"Existing bambu config found. IP: {ip} - {printerSn}") + Logger.Info("Checking if we can connect to your Bambu Labs printer...") + #success, _ = self._CheckForMoonraker(ip, port, 10.0) + success = True + if success: + Logger.Info("Successfully connected to you Bambu Labs printer!") + return + else: + # Let the user keep this connection setup, or try to set it up again. + Logger.Blank() + Logger.Warn(f"No connection found using the IP {ip}.") + if Util.AskYesOrNoQuestion("Do you want to setup the Bambu Labs printer connection again?") is False: + Logger.Info(f"Keeping the existing Bambu Labs printer connection setup. {ip} - {printerSn}") + return + + ipOrHostname, port, accessToken, printerSn = self._SetupNewBambuConnection() + ConfigHelper.WriteCompanionDetails(context, ipOrHostname, port) + ConfigHelper.WriteBambuDetails(context, accessToken, printerSn) + Logger.Blank() + Logger.Header("Bambu Connection successful!") + Logger.Blank() + + + # Helps the user setup a bambu connection via auto scanning or manual setup. + # Returns (ip:str, port:str, accessToken:str, printerSn:str) + def _SetupNewBambuConnection(self): + Logger.Blank() + Logger.Blank() + Logger.Blank() + Logger.Header("##################################") + Logger.Header(" Bambu Labs Printer Setup") + Logger.Header("##################################") + Logger.Blank() + Logger.Info("For OctoEverywhere Bambu Connect to work, it needs to know how to connect to your Bambu Labs printer.") + Logger.Info("If you have any trouble, we are happy to help! Contact us at support@octoeverywhere.com") + Logger.Blank() + ipOrHostname = input("Enter the IP or Hostname: ") + accessToken = input("Enter the Access Token: ") + printerSn = input("Enter the printer's serial number: ") + return (ipOrHostname, "8883", accessToken, printerSn) + # Logger.Info("Searching for local Klipper printers... please wait... (about 5 seconds)") + # foundIps = self._ScanForMoonrakerInstances() + # if len(foundIps) > 0: + # # Sort them so they present better. + # foundIps = sorted(foundIps) + # Logger.Blank() + # Logger.Info("Klipper was found on the following IP addresses:") + # count = 0 + # for ip in foundIps: + # count += 1 + # Logger.Info(f" {count}) {ip}:7125") + # Logger.Blank() + # while True: + # response = input("Enter the number next to the Klipper instance you want to use or enter `m` to manually setup the connection: ") + # response = response.lower().strip() + # if response == "m": + # # Break to fall through to the manual setup. + # break + # try: + # # Parse the input and -1 it, so it aligns with the array length. + # tempInt = int(response.lower().strip()) - 1 + # if tempInt >= 0 and tempInt < len(foundIps): + # return (foundIps[tempInt], "7125") + # except Exception as _: + # Logger.Warn("Invalid input, try again.") + # else: + # Logger.Info("No local Klipper devices could be automatically found.") + + # # Do the manual setup process. + # ipOrHostname = "" + # port = "7125" + # while True: + # try: + # Logger.Blank() + # Logger.Blank() + # Logger.Info("Please enter the IP address or Hostname of the device running Klipper/Moonraker/Mainsail/Fluidd.") + # Logger.Info("The IP address might look something like `192.168.1.5` or a Hostname might look like `klipper.local`") + # ipOrHostname = input("Enter the IP or Hostname: ") + # # Clean up what the user entered. + # ipOrHostname = ipOrHostname.lower().strip() + # if ipOrHostname.find("://") != -1: + # ipOrHostname = ipOrHostname[ipOrHostname.find("://")+3:] + # if ipOrHostname.find("/") != -1: + # ipOrHostname = ipOrHostname[:ipOrHostname.find("/")] + + # Logger.Blank() + # Logger.Info("Please enter the port Moonraker is running on.") + # Logger.Info("If you don't know the port or want to use the default port (7125), press enter.") + # port = input("Enter Moonraker Port: ") + # if len(port) == 0: + # port = "7125" + + # Logger.Blank() + # Logger.Info(f"Trying to connect to Moonraker via {ipOrHostname}:{port} ...") + # success, exception = self._CheckForMoonraker(ipOrHostname, port, 10.0) + + # # Handle the result. + # if success: + # return (ipOrHostname, port) + # else: + # Logger.Blank() + # Logger.Blank() + # if exception is not None: + # Logger.Error("Klipper connection failed.") + # else: + # Logger.Error("Klipper connection timed out.") + # Logger.Warn("Make sure the device is powered on, has an network connection, and the ip is correct.") + # if exception is not None: + # Logger.Warn(f"Error {str(exception)}") + # except Exception as e: + # Logger.Warn("Failed to setup Klipper, try again. "+str(e)) + + + # Given an ip or hostname and port, this will try to detect if there's a moonraker instance. + # Returns (success:, exception | None) + def _CheckForMoonraker(self, ip:str, port:str, timeoutSec:float = 5.0): + doneEvent = threading.Event() + lock = threading.Lock() + result = {} + + # Create the URL + url = f"ws://{ip}:{port}/websocket" + + # Setup the callback functions + def OnOpened(ws): + Logger.Debug(f"Test [{url}] - WS Opened") + def OnMsg(ws, msg): + with lock: + if "success" in result: + return + try: + # Try to see if the message looks like one of the first moonraker messages. + msgStr = msg.decode('utf-8') + Logger.Debug(f"Test [{url}] - WS message `{msgStr}`") + if "moonraker" in msgStr.lower(): + Logger.Debug(f"Test [{url}] - Found Moonraker message, success!") + result["success"] = True + doneEvent.set() + except Exception: + pass + def OnClosed(ws): + Logger.Debug(f"Test [{url}] - Closed") + doneEvent.set() + def OnError(ws, exception): + Logger.Debug(f"Test [{url}] - Error: {str(exception)}") + with lock: + result["exception"] = exception + doneEvent.set() + + # Create the websocket + Logger.Debug(f"Checking for moonraker using the address: `{url}`") + ws = Client(url, onWsOpen=OnOpened, onWsMsg=OnMsg, onWsError=OnError, onWsClose=OnClosed) + ws.RunAsync() + + # Wait for the event or a timeout. + doneEvent.wait(timeoutSec) + + # Get the results before we close. + capturedSuccess = False + capturedEx = None + with lock: + if result.get("success", None) is not None: + capturedSuccess = result["success"] + if result.get("exception", None) is not None: + capturedEx = result["exception"] + + # Ensure the ws is closed + try: + ws.Close() + except Exception: + pass + + return (capturedSuccess, capturedEx) + + + # Scans the subnet for Moonraker instances. + # Returns a list of IPs where moonraker was found. + def _ScanForMoonrakerInstances(self): + foundIps = [] + try: + localIp = self._TryToGetLocalIp() + if localIp is None or len(localIp) == 0: + Logger.Debug("Failed to get local IP") + return foundIps + Logger.Debug(f"Local IP found as: {localIp}") + if ":" in localIp: + Logger.Info("IPv6 addresses aren't supported for local discovery.") + return foundIps + lastDot = localIp.rfind(".") + if lastDot == -1: + Logger.Info("Failed to find last dot in local IP?") + return foundIps + ipPrefix = localIp[:lastDot+1] + + counter = 0 + doneThreads = [0] + totalThreads = 255 + threadLock = threading.Lock() + doneEvent = threading.Event() + while counter <= totalThreads: + fullIp = ipPrefix + str(counter) + def threadFunc(ip): + try: + success, _ = self._CheckForMoonraker(ip, "7125", 5.0) + with threadLock: + if success: + foundIps.append(ip) + doneThreads[0] += 1 + if doneThreads[0] == totalThreads: + doneEvent.set() + except Exception as e: + Logger.Error(f"Moonraker scan failed for {ip} "+str(e)) + t = threading.Thread(target=threadFunc, args=[fullIp]) + t.start() + counter += 1 + doneEvent.wait() + return foundIps + except Exception as e: + Logger.Error("Failed to scan for Moonraker instances. "+str(e)) + return foundIps + + + def _TryToGetLocalIp(self) -> str: + # Find the local IP. Works on Windows and Linux. Always gets the correct routable IP. + # https://stackoverflow.com/a/28950776 + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + ip = None + try: + # doesn't even have to be reachable + s.connect(('1.1.1.1', 1)) + ip = s.getsockname()[0] + except Exception: + pass + finally: + s.close() + return ip diff --git a/moonraker_installer/Configure.py b/py_installer/NetworkConnectors/MoonrakerConnector.py similarity index 51% rename from moonraker_installer/Configure.py rename to py_installer/NetworkConnectors/MoonrakerConnector.py index 11e4fb9..e2e4623 100644 --- a/moonraker_installer/Configure.py +++ b/py_installer/NetworkConnectors/MoonrakerConnector.py @@ -1,150 +1,26 @@ -import os import threading import socket from octoeverywhere.websocketimpl import Client -from .Util import Util -from .Paths import Paths -from .Logging import Logger -from .Context import Context -from .Context import OsTypes -from .ObserverConfigFile import ObserverConfigFile +from py_installer.Util import Util +from py_installer.Logging import Logger +from py_installer.Context import Context +from py_installer.ConfigHelper import ConfigHelper -# The goal of this class is the take the context object from the Discovery Gen2 phase to the Phase 3. -class Configure: - # This is the common service prefix (or word used in the file name) we use for all of our service file names. - # This MUST be used for all instances running on this device, both local plugins and companions. - # This also MUST NOT CHANGE, as it's used by the Updater logic to find all of the locally running services. - c_ServiceCommonName = "octoeverywhere" +# A class that helps the user discover, connect, and setup the details required to connect to a remote Moonraker server. +class MoonrakerConnector: - def Run(self, context:Context): - - Logger.Header("Starting configuration...") - - serviceSuffixStr = "" - if context.IsObserverSetup: - # For observers, we use the observer id, with a unique prefix to separate it from any possible local moonraker installs - # Note that the moonraker service suffix can be numbers or letters, so we use the same rules. - serviceSuffixStr = f"-companion{context.ObserverInstanceId}" - elif context.OsType == OsTypes.SonicPad: - # For Sonic Pad, we know the format of the service file is a bit different. - # For the SonicIt is moonraker_service or moonraker_service. - if "." in context.MoonrakerServiceFileName: - serviceSuffixStr = context.MoonrakerServiceFileName.split(".")[1] - elif context.OsType == OsTypes.K1: - # For the k1, there's only every one moonraker instance, so this isn't needed. - pass - else: - # Now we need to figure out the instance suffix we need to use. - # To keep with Kiauh style installs, each moonraker instances will be named moonraker-.service. - # If there is only one moonraker instance, the name is moonraker.service. - # Default to empty string, which means there's no suffix and only one instance. - serviceFileNameNoExtension = context.MoonrakerServiceFileName.split('.')[0] - if '-' in serviceFileNameNoExtension: - moonrakerServiceSuffix = serviceFileNameNoExtension.split('-') - serviceSuffixStr = "-" + moonrakerServiceSuffix[1] - Logger.Debug(f"Moonraker Service File Name: {context.MoonrakerServiceFileName}, Suffix: '{serviceSuffixStr}'") - - if context.IsObserverSetup: - # For observer setups, there is no local moonraker config file, so things are setup differently. - # The plugin data folder, which is normally the root printer data folder for that instance, becomes our per instance - # observer folder. - context.PrinterDataFolder = context.ObserverDataPath - # We mock the same layout as the moonraker folder structure, to keep things common. - context.PrinterDataConfigFolder = ObserverConfigFile.GetConfigFolderPathFromDataPath(context.PrinterDataFolder) - # Set the path to where the observer config file will be. - # This path is shared by the plugin logic, so it can't change! - context.ObserverConfigFilePath = ObserverConfigFile.GetConfigFilePathFromDataPath(context.PrinterDataFolder) - # Make a logs folder, so it's found bellow - Util.EnsureDirExists(os.path.join(context.PrinterDataFolder, "logs"), context, True) - elif context.OsType == OsTypes.SonicPad: - # ONLY FOR THE SONIC PAD, we know the folder setup is different. - # The user data folder will have /mnt/UDISK/printer_config where the config files are and /mnt/UDISK/printer_logs for logs. - # Use the normal folder for the config files. - context.PrinterDataConfigFolder = Util.GetParentDirectory(context.MoonrakerConfigFilePath) - - # There really is no printer data folder, so make one that's unique per instance. - # So based on the config folder, go to the root of it, and them make the folder "octoeverywhere_data" - context.PrinterDataFolder = os.path.join(Util.GetParentDirectory(context.PrinterDataConfigFolder), f"octoeverywhere_data{serviceSuffixStr}") - Util.EnsureDirExists(context.PrinterDataFolder, context, True) - else: - # For now we assume the folder structure is the standard Klipper folder config, - # thus the full moonraker config path will be .../something_data/config/moonraker.conf - # Based on that, we will define the config folder and the printer data root folder. - # Note that the K1 uses this standard folder layout as well. - context.PrinterDataConfigFolder = Util.GetParentDirectory(context.MoonrakerConfigFilePath) - context.PrinterDataFolder = Util.GetParentDirectory(context.PrinterDataConfigFolder) - Logger.Debug("Printer data folder: "+context.PrinterDataFolder) - - - # This is the name of our service we create. If the port is the default port, use the default name. - # Otherwise, add the port to keep services unique. - if context.OsType == OsTypes.SonicPad: - # For Sonic Pad, since the service is setup differently, follow the conventions of it. - # Both the service name and the service file name must match. - # The format is _service - # NOTE! For the Update class to work, the name must start with Configure.c_ServiceCommonNamePrefix - context.ServiceName = Configure.c_ServiceCommonName + "_service" - if len(serviceSuffixStr) != 0: - context.ServiceName= context.ServiceName + "." + serviceSuffixStr - context.ServiceFilePath = os.path.join(Paths.CrealityOsServiceFilePath, context.ServiceName) - elif context.OsType == OsTypes.K1: - # For the k1, there's only ever one moonraker and we know the exact service naming convention. - # Note we use 66 to ensure we start after moonraker. - # This is page for details on the file name: https://docs.oracle.com/cd/E36784_01/html/E36882/init.d-4.html - # Note the 'S66' string is looked for in the plugin's EnsureUpdateManagerFilesSetup function. So it must not change! - context.ServiceName = f"S66{Configure.c_ServiceCommonName}_service" - context.ServiceFilePath = os.path.join(Paths.CrealityOsServiceFilePath, context.ServiceName) - else: - # For normal setups, use the convention that Klipper users - # NOTE! For the Update class to work, the name must start with Configure.c_ServiceCommonNamePrefix - context.ServiceName = Configure.c_ServiceCommonName + serviceSuffixStr - context.ServiceFilePath = os.path.join(Paths.SystemdServiceFilePath, context.ServiceName+".service") - - # Since the moonraker config folder is unique to the moonraker instance, we will put our storage in it. - # This also prevents the user from messing with it accidentally. - context.LocalFileStorageFolder = os.path.join(context.PrinterDataFolder, "octoeverywhere-store") - - # Ensure the storage folder exists and is owned by the correct user. - Util.EnsureDirExists(context.LocalFileStorageFolder, context, True) - - # There's not a great way to find the log path from the config file, since the only place it's located is in the systemd file. - context.PrinterDataLogsFolder = None - - # First, we will see if we can find a named folder relative to this folder. - context.PrinterDataLogsFolder = os.path.join(context.PrinterDataFolder, "logs") - if os.path.exists(context.PrinterDataLogsFolder) is False: - # Try an older path - context.PrinterDataLogsFolder = os.path.join(context.PrinterDataFolder, "klipper_logs") - if os.path.exists(context.PrinterDataLogsFolder) is False: - # Try the path Creality OS uses, something like /mnt/UDISK/printer_logs - context.PrinterDataLogsFolder = os.path.join(Util.GetParentDirectory(context.PrinterDataConfigFolder), f"printer_logs{serviceSuffixStr}") - if os.path.exists(context.PrinterDataLogsFolder) is False: - # Failed, make a folder in the printer data root. - context.PrinterDataLogsFolder = os.path.join(context.PrinterDataFolder, "octoeverywhere-logs") - # Create the folder and force the permissions so our service can write to it. - Util.EnsureDirExists(context.PrinterDataLogsFolder, context, True) - - # Finally, if this is an observer setup, we need the user to tell us where moonraker is installed - # and need to write the observer config file. - if context.IsObserverSetup: - self._EnsureObserverConfigure(context) - - # Report - Logger.Info(f'Configured. Service: {context.ServiceName}, Path: {context.ServiceFilePath}, LocalStorage: {context.LocalFileStorageFolder}, Config Dir: {context.PrinterDataConfigFolder}, Logs: {context.PrinterDataLogsFolder}') - - - def _EnsureObserverConfigure(self, context:Context): - Logger.Debug("Running observer ensure config logic.") + def EnsureCompanionMoonrakerConnection(self, context:Context): + Logger.Debug("Running companion ensure config logic.") # See if there's a valid config already. - ip, port = ObserverConfigFile.TryToParseConfig(context.ObserverConfigFilePath) + ip, port = ConfigHelper.TryToGetCompanionDetails(context) if ip is not None and port is not None: # Check if we can still connect. This can happen if the IP address changes, the user might need to setup the # printer again. - Logger.Info(f"Existing observer config file found. IP: {ip}:{port}") + Logger.Info(f"Existing companion config file found. IP: {ip}:{port}") Logger.Info("Checking if we can connect to Klipper...") success, _ = self._CheckForMoonraker(ip, port, 10.0) if success: @@ -159,12 +35,12 @@ def _EnsureObserverConfigure(self, context:Context): return ip, port = self._SetupNewMoonrakerConnection() - if ObserverConfigFile.WriteIpAndPort(context, context.ObserverConfigFilePath, ip, port) is False: - raise Exception("Failed to write observer config.") + ConfigHelper.WriteCompanionDetails(context, ip, port) Logger.Blank() Logger.Header("Klipper connection successful!") Logger.Blank() + # Helps the user setup a moonraker connection via auto scanning or manual setup. # Returns (ip:str, port:str) def _SetupNewMoonrakerConnection(self): @@ -312,6 +188,7 @@ def OnError(ws, exception): return (capturedSuccess, capturedEx) + # Scans the subnet for Moonraker instances. # Returns a list of IPs where moonraker was found. def _ScanForMoonrakerInstances(self): diff --git a/moonraker_installer/Paths.py b/py_installer/Paths.py similarity index 100% rename from moonraker_installer/Paths.py rename to py_installer/Paths.py diff --git a/moonraker_installer/Permissions.py b/py_installer/Permissions.py similarity index 80% rename from moonraker_installer/Permissions.py rename to py_installer/Permissions.py index 5816275..6407065 100644 --- a/moonraker_installer/Permissions.py +++ b/py_installer/Permissions.py @@ -3,23 +3,22 @@ from .Context import Context from .Logging import Logger from .Util import Util -from .Frontend import Frontend - +from .ConfigHelper import ConfigHelper class Permissions: # Must be lower case. c_RootUserName = "root" - # For some companion setups, users only use one user on the device, root. + # For some companion or bambu setups, users only use one user on the device, root. # In this case, it's ok to install as root, but sometimes the $USER is empty # Thus, if the home user path is root, we will update the user to be root as well. # Note that since the install script always cd ~, we should have the correct home user. # # Also note, this function runs before the first context validation, so the vars could be null. def CheckUserAndCorrectIfRequired_RanBeforeFirstContextValidation(self, context:Context) -> None: - # If this is a companion install, check if we need to set the user name. + # If this is a companion or bambu install, check if we need to set the user name. # It's ok be ran as root, but sometimes the bash USER var isn't set to the user name. - if context.IsObserverSetup: + if context.IsCompanionOrBambu(): if context.UserName is None or len(context.UserName) == 0: # Since the install script does a cd ~, we know if the user home path starts with /root/, the user is root. if context.UserHomePath is not None and context.UserHomePath.lower().startswith("/root/"): @@ -28,11 +27,11 @@ def CheckUserAndCorrectIfRequired_RanBeforeFirstContextValidation(self, context: def EnsureRunningAsRootOrSudo(self, context:Context) -> None: - # IT'S NOT OK TO INSTALL AS ROOT for the normal klipper setup. + # IT'S NOT OK TO INSTALL AS ROOT for the local klipper setup. # This is because the moonraker updater system needs to get able to access the .git repo. # If the repo is owned by the root, it can't do that. # For the Sonic Pad and K1 setup, the only user is root, so it's ok. - if context.IsObserverSetup is False and context.IsCrealityOs() is False: + if context.IsCompanionOrBambu() is False and context.IsCrealityOs() is False: if context.UserName.lower() == Permissions.c_RootUserName: raise Exception("The installer was ran under the root user, this will cause problems with Moonraker. Please run the installer script as a non-root user, usually that's the `pi` user or 'mks' for MKS PI.") @@ -67,9 +66,10 @@ def SetPermissions(path:str): Util.SetFileOwnerRecursive(context.RepoRootFolder, context.UserName) # These following files or folders must be owned by the user the service is running under. - f = Frontend() - SetPermissions(f.GetOctoEverywhereServiceConfigFilePath(context)) - SetPermissions(context.MoonrakerConfigFilePath) - SetPermissions(context.ObserverDataPath) + # Ensure the the main plugin config - this is important because if this installer made the config, it's owned by root not the service user. + SetPermissions(ConfigHelper.GetConfigFilePath(context)) + # The folder where we will store files. SetPermissions(context.LocalFileStorageFolder) - SetPermissions(context.ObserverConfigFilePath) + # If this is a companion or bambu setup, the root data folder + SetPermissions(context.CompanionDataRoot) + # Note we can't set permission on the log folder, since there are logs in there that other services own. diff --git a/moonraker_installer/ReadMe.py b/py_installer/ReadMe.py similarity index 100% rename from moonraker_installer/ReadMe.py rename to py_installer/ReadMe.py diff --git a/moonraker_installer/Service.py b/py_installer/Service.py similarity index 85% rename from moonraker_installer/Service.py rename to py_installer/Service.py index 9154034..85548a9 100644 --- a/moonraker_installer/Service.py +++ b/py_installer/Service.py @@ -22,35 +22,44 @@ def Install(self, context:Context): # First, we create a json object that we use as arguments. Using a json object makes parsing and such more flexible. # We base64 encode the json string to prevent any arg passing issues with things like quotes, spaces, or other chars. - # Note some of these vars might be null, in the Observer Setup case - argsJson = json.dumps({ - 'KlipperConfigFolder': context.PrinterDataConfigFolder, - 'MoonrakerConfigFile': context.MoonrakerConfigFilePath, - 'KlipperLogFolder': context.PrinterDataLogsFolder, + # Note some of these vars might be null, in the companion Setup case + args = { + 'ConfigFolder': context.ConfigFolder, + 'LogFolder': context.LogsFolder, 'LocalFileStoragePath': context.LocalFileStorageFolder, 'ServiceName': context.ServiceName, 'VirtualEnvPath': context.VirtualEnvPath, 'RepoRootFolder': context.RepoRootFolder, - 'IsObserver' : context.IsObserverSetup, - 'ObserverConfigFilePath' : context.ObserverConfigFilePath, - 'ObserverInstanceIdStr' : context.ObserverInstanceId - }) + 'IsCompanion' : context.IsCompanionSetup, + } + # Set plugin specific vars. + # These vars are used on all companion and bambu setups. + if context.IsCompanionOrBambu(): + args['CompanionInstanceIdStr'] = context.CompanionInstanceId + # These vars are used on anything that's NOT a companion or bambu + if not context.IsCompanionOrBambu(): + args['MoonrakerConfigFile'] = context.MoonrakerConfigFilePath + # We have to convert to bytes -> encode -> back to string. + argsJson = json.dumps(args) argsJsonBase64 = base64.urlsafe_b64encode(bytes(argsJson, "utf-8")).decode("utf-8") + # Get the correct module host for the service to run, based on the install type. + moduleNameToRun = "bambu_octoeverywhere" if context.IsBambuSetup else "moonraker_octoeverywhere" + # Base on the OS type, install the service differently if context.OsType == OsTypes.Debian: - self._InstallDebian(context, argsJsonBase64) + self._InstallDebian(context, argsJsonBase64, moduleNameToRun) elif context.OsType == OsTypes.SonicPad: - self._InstallSonicPad(context, argsJsonBase64) + self._InstallSonicPad(context, argsJsonBase64, moduleNameToRun) elif context.OsType == OsTypes.K1: - self._InstallK1(context, argsJsonBase64) + self._InstallK1(context, argsJsonBase64, moduleNameToRun) else: raise Exception("Service install is not supported for this OS type yet. Contact support!") # Install for debian setups - def _InstallDebian(self, context:Context, argsJsonBase64): + def _InstallDebian(self, context:Context, argsJsonBase64:str, moduleNameToRun:str): s = f'''\ # OctoEverywhere For Moonraker Service [Unit] @@ -66,7 +75,7 @@ def _InstallDebian(self, context:Context, argsJsonBase64): Type=simple User={context.UserName} WorkingDirectory={context.RepoRootFolder} - ExecStart={context.VirtualEnvPath}/bin/python3 -m moonraker_octoeverywhere "{argsJsonBase64}" + ExecStart={context.VirtualEnvPath}/bin/python3 -m {moduleNameToRun} "{argsJsonBase64}" Restart=always # Since we will only restart on a fatal Logger.Error, set the restart time to be a bit higher, so we don't spin and spam. RestartSec=10 @@ -92,7 +101,7 @@ def _InstallDebian(self, context:Context, argsJsonBase64): # Install for sonic pad setups. - def _InstallSonicPad(self, context:Context, argsJsonBase64): + def _InstallSonicPad(self, context:Context, argsJsonBase64:str, moduleNameToRun:str): # First, write the service file # Notes: # Set start to be 66, so we start after Moonraker. @@ -113,7 +122,7 @@ def _InstallSonicPad(self, context:Context, argsJsonBase64): procd_set_param env HOME=/root procd_set_param env PYTHONPATH={context.RepoRootFolder} procd_set_param oom_adj $OOM_ADJ - procd_set_param command {context.VirtualEnvPath}/bin/python3 -m moonraker_octoeverywhere "{argsJsonBase64}" + procd_set_param command {context.VirtualEnvPath}/bin/python3 -m {moduleNameToRun} "{argsJsonBase64}" procd_close_instance }} ''' @@ -137,7 +146,7 @@ def _InstallSonicPad(self, context:Context, argsJsonBase64): # Install for k1 and k1 max - def _InstallK1(self, context:Context, argsJsonBase64): + def _InstallK1(self, context:Context, argsJsonBase64:str, moduleNameToRun:str): # On the K1 start-stop-daemon is used to run services. # But, to launch our service, we have to use the py module run, which requires a environment var to be # set for PYTHONPATH. The command can't set the env, so we write this script to our store, where we then run @@ -152,7 +161,7 @@ def _InstallK1(self, context:Context, argsJsonBase64): # # Don't edit this script, it's generated by the ./install.sh script during the OE install and update.. # -PYTHONPATH={context.RepoRootFolder} {context.VirtualEnvPath}/bin/python3 -m moonraker_octoeverywhere "{argsJsonBase64}" +PYTHONPATH={context.RepoRootFolder} {context.VirtualEnvPath}/bin/python3 -m {moduleNameToRun} "{argsJsonBase64}" exit $? ''' # Write the required service file, make it point to our run script. diff --git a/moonraker_installer/TimeSync.py b/py_installer/TimeSync.py similarity index 93% rename from moonraker_installer/TimeSync.py rename to py_installer/TimeSync.py index 6bb4d5c..7b82489 100644 --- a/moonraker_installer/TimeSync.py +++ b/py_installer/TimeSync.py @@ -1,6 +1,8 @@ from .Util import Util from .Logging import Logger +from .Context import Context + # This helper class ensures that the system's ntp clock sync service is enabled and active. # We found some MKS PI systems didn't have it on, and would be years out of sync on reboot. # This is a problem because SSL will fail if the date is too far out of sync. @@ -10,7 +12,10 @@ class TimeSync: @staticmethod - def EnsureNtpSyncEnabled(): + def EnsureNtpSyncEnabled(context:Context): + if context.SkipSudoActions: + Logger.Warn("Skipping time sync since we are skipping sudo actions.") + return Logger.Info("Ensuring that time sync is enabled...") # Ensure that NTP is uninstalled, since this conflicts with timesyncd diff --git a/moonraker_installer/Uninstall.py b/py_installer/Uninstall.py similarity index 92% rename from moonraker_installer/Uninstall.py rename to py_installer/Uninstall.py index 8f4b8cc..9941667 100644 --- a/moonraker_installer/Uninstall.py +++ b/py_installer/Uninstall.py @@ -11,34 +11,33 @@ class Uninstall: def DoUninstall(self, context:Context): + # Ensure there's something to remove first. + # Since all service names must use the same identifier in them, we can find any services using the same search. + foundOeServices = [] + fileAndDirList = sorted(os.listdir(Paths.GetServiceFileFolderPath(context))) + for fileOrDirName in fileAndDirList: + Logger.Debug(f"Searching for OE services to remove, found: {fileOrDirName}") + if Configure.c_ServiceCommonName in fileOrDirName.lower(): + foundOeServices.append(fileOrDirName) + if len(foundOeServices) == 0: + Logger.Warn("No local, companion, or Bambu Connect plugins were found to remove.") + return Logger.Blank() Logger.Blank() - Logger.Header("You're about to uninstall OctoEverywhere.") - Logger.Info ("This printer ID will be deleted, but you can always reinstall the plugin and re-add this printer.") + Logger.Header("You're about to uninstall ALL OctoEverywhere local, companion, or Bambu Connect plugins on this device.") + Logger.Info ("This printer ID(s) will be deleted, but you can always reinstall the plugin and re-add this printer.") Logger.Blank() - r = input("Are you want to uninstall? [y/n]") + r = input("Are you want to uninstall ALL plugins? [y/n]") r = r.lower().strip() if r != "y": - Logger.Info("Uninstall canceled.") + Logger.Info("Canceled.") Logger.Blank() return Logger.Blank() Logger.Blank() Logger.Header("Starting OctoEverywhere uninstall") - # Since all service names must use the same identifier in them, we can find any services using the same search. - foundOeServices = [] - fileAndDirList = sorted(os.listdir(Paths.GetServiceFileFolderPath(context))) - for fileOrDirName in fileAndDirList: - Logger.Debug(f" Searching for OE services to remove, found: {fileOrDirName}") - if Configure.c_ServiceCommonName in fileOrDirName.lower(): - foundOeServices.append(fileOrDirName) - - if len(foundOeServices) == 0: - Logger.Warn("No local plugins or companions were found to remove.") - return - # TODO - We need to cleanup more, but for now, just make sure any services are shutdown. Logger.Info("Stopping services...") for serviceFileName in foundOeServices: diff --git a/moonraker_installer/Updater.py b/py_installer/Updater.py similarity index 92% rename from moonraker_installer/Updater.py rename to py_installer/Updater.py index e00d26b..95613b6 100644 --- a/moonraker_installer/Updater.py +++ b/py_installer/Updater.py @@ -1,7 +1,7 @@ import os import stat -from moonraker_octoeverywhere.version import Version +from linux_host.version import Version from .Context import Context from .Context import OsTypes @@ -12,7 +12,7 @@ from .Util import Util # -# This class is responsible for doing updates for all plugins and companions on this local system. +# This class is responsible for doing updates for all local, companions, and bambu connect plugins on this local system. # This update logic is mostly for companion plugins, since normal plugins will be updated via the moonraker update system. # But it does work for both. # @@ -29,19 +29,19 @@ class Updater: def DoUpdate(self, context:Context): Logger.Header("Starting Update Logic") - # Enumerate all service file to find any local plugins, Sonic Pad plugins, and companion service files, since all service files contain this name. + # Enumerate all service file to find any local plugins, Sonic Pad plugins, companion service files, and bambu service files, since all service files contain this name. # Note GetServiceFileFolderPath will return dynamically based on the OsType detected. # Use sorted, so the results are in a nice user presentable order. foundOeServices = [] fileAndDirList = sorted(os.listdir(Paths.GetServiceFileFolderPath(context))) for fileOrDirName in fileAndDirList: - Logger.Debug(f" Searching for OE services to update, found: {fileOrDirName}") + Logger.Debug(f"Searching for OE services to update, found: {fileOrDirName}") if Configure.c_ServiceCommonName in fileOrDirName.lower(): foundOeServices.append(fileOrDirName) if len(foundOeServices) == 0: - Logger.Warn("No local plugins or companions were found.") - raise Exception("No local plugins or companions were found.") + Logger.Warn("No local, companion, or Bambu Connect plugins were found on this device.") + raise Exception("No local, companion, or Bambu Connect plugins were found on this device.") Logger.Info("We found the following plugins to update:") for s in foundOeServices: @@ -97,9 +97,9 @@ def PlaceUpdateScriptInRoot(self, context:Context) -> bool: #!/bin/bash # -# Run this script to update all OctoEverywhere instances on this device! +# Run this script to update all OctoEverywhere plugins on this device! # -# This works for all install types, normal plugins, Creality OS, and companion installs. +# This works for all plugin types, such as local Klipper, Creality OS, Companion, and Bambu Connect. # # If you need help, feel free to contact us at support@octoeverywhere.com # diff --git a/moonraker_installer/Util.py b/py_installer/Util.py similarity index 100% rename from moonraker_installer/Util.py rename to py_installer/Util.py diff --git a/py_installer/__init__.py b/py_installer/__init__.py new file mode 100644 index 0000000..93c04b5 --- /dev/null +++ b/py_installer/__init__.py @@ -0,0 +1 @@ +# Note the module name is "py_installer" instead of "installer" so ./install.sh doesn't get conflicting auto completes. diff --git a/moonraker_installer/__main__.py b/py_installer/__main__.py similarity index 100% rename from moonraker_installer/__main__.py rename to py_installer/__main__.py diff --git a/requirements.txt b/requirements.txt index cbdc41f..408a3fb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,4 +17,6 @@ dnspython>=2.3.0 httpx>=0.24.1,<0.26.0 urllib3>=1.26.15,<2.0.0 # The following are required only for Moonraker -configparser \ No newline at end of file +configparser +# Only used for Bambu Connect +paho-mqtt>=2.0.0 \ No newline at end of file From 6dc1b24357603e80d0bc342eb65697b2d05cd183 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Tue, 27 Feb 2024 20:06:02 -0800 Subject: [PATCH 029/328] Version bump --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bcc418a..6e83269 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this is also parsed by the moonraker module to pull the version, so the string and format must remain the same! -plugin_version = "2.10.9" +plugin_version = "2.2.0" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From b181289a6b48cf18f87ba5b49cb520aa9d47ef80 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Tue, 27 Feb 2024 20:10:55 -0800 Subject: [PATCH 030/328] Oops, this is the proper version bump. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6e83269..77cdf10 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this is also parsed by the moonraker module to pull the version, so the string and format must remain the same! -plugin_version = "2.2.0" +plugin_version = "2.11.0" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From f4d2afec975e2ba56a9a5fa15320da52617d4d28 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Thu, 29 Feb 2024 12:58:57 -0800 Subject: [PATCH 031/328] Minor fix for crealtiy printers. --- py_installer/Configure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py_installer/Configure.py b/py_installer/Configure.py index c744f1c..58cb3e2 100644 --- a/py_installer/Configure.py +++ b/py_installer/Configure.py @@ -119,7 +119,7 @@ def Run(self, context:Context): context.LogsFolder = os.path.join(context.RootFolder, "klipper_logs") if os.path.exists(context.LogsFolder) is False: # Try the path Creality OS uses, something like /mnt/UDISK/printer_logs - context.LogsFolder = os.path.join(Util.GetParentDirectory(context.PrinterDataConfigFolder), f"printer_logs{serviceSuffixStr}") + context.LogsFolder = os.path.join(Util.GetParentDirectory(context.RootFolder), f"printer_logs{serviceSuffixStr}") if os.path.exists(context.LogsFolder) is False: # Failed, make a folder in the printer data root. context.LogsFolder = os.path.join(context.RootFolder, "octoeverywhere-logs") From 769a21fe8926b4baa6e8e3342f1dc9ae94324999 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Fri, 1 Mar 2024 20:21:50 -0800 Subject: [PATCH 032/328] Adding a small log to an error message. --- linux_host/config.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/linux_host/config.py b/linux_host/config.py index f04a20e..4776dc0 100644 --- a/linux_host/config.py +++ b/linux_host/config.py @@ -142,6 +142,7 @@ def GetStr(self, section, key, defaultValue) -> str: # If the default value is None, the default will not be written into the config. def GetInt(self, section:str, key:str, defaultValue) -> int: # Use a try catch, so if a user sets an invalid value, it doesn't crash us. + result = None try: # If None is passed as the default, don't str it. if defaultValue is not None: @@ -154,7 +155,7 @@ def GetInt(self, section:str, key:str, defaultValue) -> int: return int(str) except Exception as e: - self.Logger.error("Config settings error! "+key+" failed to get as int. Resetting to default. "+str(e)) + self.Logger.error(f"Config settings error! {key} failed to get as int. Value was `{result}`. Resetting to default. "+str(e)) self.SetStr(section, key, str(defaultValue)) return int(defaultValue) @@ -164,6 +165,7 @@ def GetInt(self, section:str, key:str, defaultValue) -> int: # If the default value is None, the default will not be written into the config. def GetBool(self, section, key, defaultValue) -> bool: # Use a try catch, so if a user sets an invalid value, it doesn't crash us. + result = None try: # If None is passed as the default, don't str it. if defaultValue is not None: @@ -182,7 +184,7 @@ def GetBool(self, section, key, defaultValue) -> bool: return True raise Exception("Invalid bool value, value was: "+strValue) except Exception as e: - self.Logger.error("Config settings error! "+key+" failed to get as bool. Resetting to default. "+str(e)) + self.Logger.error(f"Config settings error! {key} failed to get as bool. Value was `{result}`. Resetting to default. "+str(e)) self.SetStr(section, key, str(defaultValue)) return bool(defaultValue) From 10ee3c21161c9d89dd2184572f6a1222689fa58f Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sat, 2 Mar 2024 12:45:51 -0800 Subject: [PATCH 033/328] Fixing config bug --- linux_host/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linux_host/config.py b/linux_host/config.py index 4776dc0..17eee26 100644 --- a/linux_host/config.py +++ b/linux_host/config.py @@ -153,7 +153,7 @@ def GetInt(self, section:str, key:str, defaultValue) -> int: if result is None: return None - return int(str) + return int(result) except Exception as e: self.Logger.error(f"Config settings error! {key} failed to get as int. Value was `{result}`. Resetting to default. "+str(e)) self.SetStr(section, key, str(defaultValue)) From 4d0947b577a04d83aa3df8dc1c88853d090359e8 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sat, 2 Mar 2024 13:03:15 -0800 Subject: [PATCH 034/328] Adding the Bambu Labs host type. --- octoeverywhere/Proto/ServerHost.py | 1 + 1 file changed, 1 insertion(+) diff --git a/octoeverywhere/Proto/ServerHost.py b/octoeverywhere/Proto/ServerHost.py index 1ea3d84..340d5a0 100644 --- a/octoeverywhere/Proto/ServerHost.py +++ b/octoeverywhere/Proto/ServerHost.py @@ -6,4 +6,5 @@ class ServerHost(object): Unknown = 0 OctoPrint = 1 Moonraker = 2 + Bambu = 3 From 2b89a9cc35b067a9dea4f0929a946c24745ad0b9 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Tue, 5 Mar 2024 21:39:58 -0800 Subject: [PATCH 035/328] Major changes getting ready for Bambu Connect! --- .pylintrc | 1 + .vscode/launch.json | 6 +- .vscode/settings.json | 10 + bambu_octoeverywhere/__main__.py | 4 +- bambu_octoeverywhere/bambuclient.py | 349 ++++++++++++++-- bambu_octoeverywhere/bambucommandhandler.py | 328 ++++++--------- bambu_octoeverywhere/bambuhost.py | 45 +- bambu_octoeverywhere/bambumodels.py | 158 +++++++ bambu_octoeverywhere/bambustatetranslater.py | 261 ++++++++++++ bambu_octoeverywhere/bambuwebcamhelper.py | 144 +++++-- bambu_octoeverywhere/quickcam.py | 224 ++++++++++ developer/runpylint.cmd | 21 +- developer/runpylint.sh | 18 +- linux_host/networksearch.py | 198 +++++++++ moonraker_octoeverywhere/moonrakerclient.py | 2 +- .../WebStream/octowebstreamhttphelper.py | 116 +++--- .../WebStream/octowebstreamwshelper.py | 4 + octoeverywhere/commandhandler.py | 29 +- octoeverywhere/notificationshandler.py | 28 +- octoeverywhere/octohttprequest.py | 135 ++++-- octoeverywhere/requestsutils.py | 18 - octoeverywhere/webcamhelper.py | 79 ++-- octoprint_octoeverywhere/slipstream.py | 29 +- py_installer/Logging.py | 28 +- .../NetworkConnectors/BambuConnector.py | 390 ++++++++---------- py_installer/Permissions.py | 2 +- py_installer/Service.py | 12 +- setup.py | 2 +- 28 files changed, 1934 insertions(+), 707 deletions(-) create mode 100644 bambu_octoeverywhere/bambumodels.py create mode 100644 bambu_octoeverywhere/bambustatetranslater.py create mode 100644 bambu_octoeverywhere/quickcam.py create mode 100644 linux_host/networksearch.py delete mode 100644 octoeverywhere/requestsutils.py diff --git a/.pylintrc b/.pylintrc index 30f41eb..09fc441 100644 --- a/.pylintrc +++ b/.pylintrc @@ -24,6 +24,7 @@ disable= R0916, # Too many booleans in one if statment. R1730, # consider-using-min-builtin R1731, # consider-using-max-builtin + R1724, # Unnecessary "else" after "break" # A comma-separated list of package or module names from where C extensions may diff --git a/.vscode/launch.json b/.vscode/launch.json index 4b0a59c..7c6234d 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -44,8 +44,8 @@ "args": [ // These args reflect the correct setup for a pi installed with the Bambu Connect version of the plugin. These args target the first instance. // - // { "ServiceName": "octoeverywhere-bambu", "VirtualEnvPath":"/home/pi/octoeverywhere-env", "RepoRootFolder":"/home/pi/octoeverywhere/", "LogFolder":"/home/pi/.octoeverywhere-bambu/logs", "LocalFileStoragePath":"/home/pi/.octoeverywhere-bambu/octoeverywhere-store", "ConfigFolder":"/home/pi/.octoeverywhere-bambu/", "InstanceStr":"1" } - "eyAiU2VydmljZU5hbWUiOiAib2N0b2V2ZXJ5d2hlcmUtYmFtYnUiLCAiVmlydHVhbEVudlBhdGgiOiIvaG9tZS9waS9vY3RvZXZlcnl3aGVyZS1lbnYiLCAiUmVwb1Jvb3RGb2xkZXIiOiIvaG9tZS9waS9vY3RvZXZlcnl3aGVyZS8iLCAiTG9nRm9sZGVyIjoiL2hvbWUvcGkvLm9jdG9ldmVyeXdoZXJlLWJhbWJ1L2xvZ3MiLCAiTG9jYWxGaWxlU3RvcmFnZVBhdGgiOiIvaG9tZS9waS8ub2N0b2V2ZXJ5d2hlcmUtYmFtYnUvb2N0b2V2ZXJ5d2hlcmUtc3RvcmUiLCAiQ29uZmlnRm9sZGVyIjoiL2hvbWUvcGkvLm9jdG9ldmVyeXdoZXJlLWJhbWJ1LyIsICJJbnN0YW5jZVN0ciI6IjEiIH0=", + // { "ServiceName": "octoeverywhere-bambu", "VirtualEnvPath":"/home/pi/octoeverywhere-env", "RepoRootFolder":"/home/pi/octoeverywhere/", "LogFolder":"/home/pi/.octoeverywhere-bambu/logs", "LocalFileStoragePath":"/home/pi/.octoeverywhere-bambu/octoeverywhere-store", "ConfigFolder":"/home/pi/.octoeverywhere-bambu/", "CompanionInstanceIdStr":"1" } + "eyAiU2VydmljZU5hbWUiOiAib2N0b2V2ZXJ5d2hlcmUtYmFtYnUiLCAiVmlydHVhbEVudlBhdGgiOiIvaG9tZS9waS9vY3RvZXZlcnl3aGVyZS1lbnYiLCAiUmVwb1Jvb3RGb2xkZXIiOiIvaG9tZS9waS9vY3RvZXZlcnl3aGVyZS8iLCAiTG9nRm9sZGVyIjoiL2hvbWUvcGkvLm9jdG9ldmVyeXdoZXJlLWJhbWJ1L2xvZ3MiLCAiTG9jYWxGaWxlU3RvcmFnZVBhdGgiOiIvaG9tZS9waS8ub2N0b2V2ZXJ5d2hlcmUtYmFtYnUvb2N0b2V2ZXJ5d2hlcmUtc3RvcmUiLCAiQ29uZmlnRm9sZGVyIjoiL2hvbWUvcGkvLm9jdG9ldmVyeXdoZXJlLWJhbWJ1LyIsICJDb21wYW5pb25JbnN0YW5jZUlkU3RyIjoiMSIgfQ==", // We can optionally pass a dev config json object, which has dev specific overwrites we can make. "{\"LocalServerAddress\":\"\", \"LogLevel\":\"DEBUG\"}" ] @@ -104,7 +104,7 @@ // The module requires this json object to be passed. // Normally the install.sh script runs, ensure everything is installed, creates a virtural env, and then runs this modlue giving it these args. // But for debugging, we can skip that assuming it's already been ran. - "{\"OE_REPO_DIR\":\"/home/pi/octoeverywhere\",\"OE_ENV\":\"/home/pi/octoeverywhere-env\",\"USERNAME\":\"pi\",\"USER_HOME\":\"/home/pi\",\"CMD_LINE_ARGS\":\"-debug -skipsudoactions -bambu\"}" + "{\"OE_REPO_DIR\":\"/home/pi/octoeverywhere\",\"OE_ENV\":\"/home/pi/octoeverywhere-env\",\"USERNAME\":\"pi\",\"USER_HOME\":\"/home/pi\",\"CMD_LINE_ARGS\":\"-skipsudoactions -bambu\"}" ] }, { diff --git a/.vscode/settings.json b/.vscode/settings.json index 92773f3..57ac3cc 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -3,16 +3,20 @@ "cSpell.words": [ "ABNF", "accesscontrol", + "acked", "Analysing", "apicommandhandler", "appi", "asvc", "authed", + "autobedlevel", "backoff", "Bambu", "bambuclient", "bambucommandhandler", "bambuhost", + "bambumodels", + "bambustatetranslater", "bambuwebcamhelper", "bblp", "bootstraper", @@ -22,6 +26,7 @@ "certifi", "checkin", "classicwebcam", + "cleaningnozzle", "commandhandler", "Commonize", "comms", @@ -73,11 +78,13 @@ "journalctl", "JRPC", "jsonify", + "keepalive", "keyvalidator", "KIAUH", "Klipper", "Klippy", "levelname", + "levelno", "localauth", "localfs", "localip", @@ -100,6 +107,7 @@ "multicam", "myprinter", "nbsp", + "networksearch", "noatuoselect", "notificationshandler", "Ocoto", @@ -127,6 +135,7 @@ "octowebstreamhttphelperimpl", "octowebstreamimpl", "octowebstreamwshelper", + "oestreamboundary", "openwrt", "opkg", "oprint", @@ -151,6 +160,7 @@ "Pylint", "pythoncompat", "PYTHONPATH", + "quickcam", "ratos", "rdataclass", "rdclass", diff --git a/bambu_octoeverywhere/__main__.py b/bambu_octoeverywhere/__main__.py index 5ea9c23..3a1b4ae 100644 --- a/bambu_octoeverywhere/__main__.py +++ b/bambu_octoeverywhere/__main__.py @@ -17,7 +17,7 @@ jsonConfig = s.GetJsonFromArgs(sys.argv) # - # 1) Parse the common, required args. + # Parse the common, required args. # ServiceName = s.GetConfigVarAndValidate(jsonConfig, "ServiceName", ConfigDataTypes.String) VirtualEnvPath = s.GetConfigVarAndValidate(jsonConfig, "VirtualEnvPath", ConfigDataTypes.Path) @@ -25,7 +25,7 @@ LocalFileStoragePath = s.GetConfigVarAndValidate(jsonConfig, "LocalFileStoragePath", ConfigDataTypes.Path) LogFolder = s.GetConfigVarAndValidate(jsonConfig, "LogFolder", ConfigDataTypes.Path) ConfigFolder = s.GetConfigVarAndValidate(jsonConfig, "ConfigFolder", ConfigDataTypes.Path) - InstanceStr = s.GetConfigVarAndValidate(jsonConfig, "InstanceStr", ConfigDataTypes.String) + InstanceStr = s.GetConfigVarAndValidate(jsonConfig, "CompanionInstanceIdStr", ConfigDataTypes.String) except Exception as e: s.PrintErrorAndExit(f"Exception while loading json config. Error:{str(e)}, Config: {jsonConfigStr}") diff --git a/bambu_octoeverywhere/bambuclient.py b/bambu_octoeverywhere/bambuclient.py index 1f1245f..a149518 100644 --- a/bambu_octoeverywhere/bambuclient.py +++ b/bambu_octoeverywhere/bambuclient.py @@ -2,58 +2,349 @@ import ssl import json import threading +import time +from typing import List import paho.mqtt.client as mqtt +from octoeverywhere.sentry import Sentry + from linux_host.config import Config +from linux_host.networksearch import NetworkSearch + +from .bambumodels import BambuState, BambuVersion +# Responsible for connecting to and maintaining a connection to the Bambu Printer. +# Also responsible for dispatching out MQTT update messages. class BambuClient: - def __init__(self, logger:logging.Logger, config:Config) -> None: + _Instance = None + + @staticmethod + def Init(logger:logging.Logger, config:Config, stateTranslator): + BambuClient._Instance = BambuClient(logger, config, stateTranslator) + + + @staticmethod + def Get(): + return BambuClient._Instance + + + def __init__(self, logger:logging.Logger, config:Config, stateTranslator) -> None: self.Logger = logger + self.StateTranslator = stateTranslator # BambuStateTranslator + + # Used to keep track of the printer state + # None means we are disconnected. + self.State:BambuState = None + self.Version:BambuVersion = None + self.HasDoneFirstFullStateSync = False + self.ReportSubscribeMid = None + self._CleanupStateOnDisconnect() - self.IpOrHostname = config.GetStr(Config.SectionCompanion, Config.CompanionKeyIpOrHostname, None) + # Get the required args. + self.Config = config + ipOrHostname = config.GetStr(Config.SectionCompanion, Config.CompanionKeyIpOrHostname, None) self.PortStr = config.GetStr(Config.SectionCompanion, Config.CompanionKeyPort, None) self.AccessToken = config.GetStr(Config.SectionBambu, Config.BambuAccessToken, None) self.PrinterSn = config.GetStr(Config.SectionBambu, Config.BambuPrinterSn, None) - if self.IpOrHostname is None or self.PortStr is None or self.AccessToken is None or self.PrinterSn is None: + if ipOrHostname is None or self.PortStr is None or self.AccessToken is None or self.PrinterSn is None: raise Exception("Missing required args from the config") - self.client = mqtt.Client(mqtt.CallbackAPIVersion.VERSION2) - self.client.tls_set(tls_version=ssl.PROTOCOL_TLS, cert_reqs=ssl.CERT_NONE) - self.client.tls_insecure_set(True) - self.client.username_pw_set("bblp", self.AccessToken) - self.client.on_connect = self.OnConnect - self.client.on_message = self.OnMessage - self.client.on_disconnect = self.OnDisconnect - self.client.connect(self.IpOrHostname, int(self.PortStr), 60) - t = threading.Thread(target=self.Worker) + # We use this var to keep track of consecutively failed connections + self.ConsecutivelyFailedConnectionAttempts = 0 + + # Start a thread to setup and maintain the connection. + self.Client:mqtt.Client = None + t = threading.Thread(target=self._ClientWorker) + t.start() + + + # Returns the current local State object which is kept in sync with the printer. + # Returns None if the printer is not connected and the state is unknown. + def GetState(self) -> BambuState: + return self.State + + + # Returns the current local Version object which is kept in sync with the printer. + # Returns None if the printer is not connected and the state is unknown. + def GetVersion(self) -> BambuVersion: + return self.Version + + + # Sends the pause command, returns is the send was successful or not. + def SendPause(self) -> bool: + return self._Publish({"print": {"sequence_id": "0", "command": "pause"}}) + + + # Sends the resume command, returns is the send was successful or not. + def SendResume(self) -> bool: + return self._Publish({"print": {"sequence_id": "0", "command": "resume"}}) + + + # Sends the cancel (stop) command, returns is the send was successful or not. + def SendCancel(self) -> bool: + return self._Publish({"print": {"sequence_id": "0", "command": "stop"}}) + + + # Sets up, runs, and maintains the MQTT connection. + def _ClientWorker(self): + localBackoffCounter = 0 + while True: + ipOrHostname = None + try: + # We always connect locally. We use encryption, but the printer doesn't have a trusted + # cert root, so we have to disable the cert root checks. + self.Client = mqtt.Client(mqtt.CallbackAPIVersion.VERSION2) + self.Client.tls_set(tls_version=ssl.PROTOCOL_TLS, cert_reqs=ssl.CERT_NONE) + self.Client.tls_insecure_set(True) + self.Client.username_pw_set("bblp", self.AccessToken) + + # Setup the callback functions. + self.Client.on_connect = self._OnConnect + self.Client.on_message = self._OnMessage + self.Client.on_disconnect = self._OnDisconnect + self.Client.on_subscribe = self._OnSubscribe + + # Get the IP to try on this connect + ipOrHostname = self._GetIpOrHostnameToTry() + + # Connect to the server + # This will throw if it fails, but after that, the loop_forever will handle reconnecting. + localBackoffCounter += 1 + self.Client.connect(ipOrHostname, int(self.PortStr), keepalive=60) + + # Note that self.Client.connect will not throw if there's no MQTT server, but not if auth is wrong. + # So if it didn't throw, we know there's a server there, but it might not be the right server + localBackoffCounter = 0 + + # This will run forever, including handling reconnects and such. + self.Client.loop_forever() + + except ConnectionRefusedError as e: + # This means there was no open socket at the given IP and port. + self.Logger.error(f"Failed to connect to the Bambu printer {ipOrHostname}:{self.PortStr}, we will retry in a bit. "+str(e)) + except TimeoutError as e: + # This means there was no open socket at the given IP and port. + self.Logger.error(f"Failed to connect to the Bambu printer {ipOrHostname}:{self.PortStr}, we will retry in a bit. "+str(e)) + except Exception as e: + # Random other errors. + Sentry.Exception("Failed to connect to Bambu printer.", e) + + # Sleep for a bit between tries. + # The main consideration here is to not log too much when the printer is off. But we do still want to connect quickly, when it's back on. + localBackoffCounter = min(localBackoffCounter, 5) + time.sleep(5 * localBackoffCounter) + + + # Since MQTT sends a full state and then partial updates, we sometimes need to force a full state sync, like on connect. + # This must be done async for most callers, since it blocks until the publish is acked. If this blocked on the main mqtt thread, it would + # dead lock. + # If this fails, it will disconnect the client. + def _ForceStateSyncAsync(self) -> bool: + def _FullSyncWorker(): + try: + self.Logger.info("Starting full state sync.") + # It's important to request the hardware version first, so we have it parsed before we get the first full sync. + getInfo = {"info": {"sequence_id": "0", "command": "get_version"}} + if not self._Publish(getInfo): + raise Exception("Failed to publish get_version") + pushAll = { "pushing": {"sequence_id": "55", "command": "pushall"}} + if not self._Publish(pushAll): + raise Exception("Failed to publish full sync") + except Exception as e: + # Report and disconnect since we are in an unknown state. + Sentry.Exception("BambuClient _ForceStateSyncAsync exception.", e) + self.Client.disconnect() + t = threading.Thread(target=_FullSyncWorker) t.start() - def DoesNothing(self): - pass + # Fired whenever the client is disconnected, we need to clean up the state since it's now unknown. + def _CleanupStateOnDisconnect(self): + self.State = None + self.Version = None + self.HasDoneFirstFullStateSync = False + self.ReportSubscribeMid = None + + + # Fired when the MQTT connection is made. + def _OnConnect(self, client:mqtt.Client, userdata, flags, reason_code, properties): + self.Logger.info("Connection to the Bambu printer established! - Subscribing to the report subscription.") + # After connect, we try to subscribe to the report feed. + # We must do this before anything else, otherwise we won't get responses for things like + # the full state sync. The result of the subscribe will be reported to _OnSubscribe + # Note that at least for my P1P, if the SN is incorrect, the MQTT connection is closed with no _OnSubscribe callback. + (result, self.ReportSubscribeMid) = self.Client.subscribe(f"device/{self.PrinterSn}/report") + if result != mqtt.MQTT_ERR_SUCCESS or self.ReportSubscribeMid is None: + # If we can't sub, disconnect, since we can't do anything. + self.Client.disconnect() - def Worker(self): - self.client.loop_forever() + # Fired when the MQTT connection is lost + def _OnDisconnect(self, client, userdata, disconnect_flags, reason_code, properties): + self.Logger.warn("Bambu printer connection lost. We will try to reconnect in a few seconds.") + # Clear the state since we lost the connection and won't stay synced. + self._CleanupStateOnDisconnect() - def OnConnect(self, client, userdata, flags, reason_code, properties): - self.Logger.warn("MQTT connected") - client.subscribe(f"device/{self.PrinterSn}/report") + # Fried when the MQTT subscribe result has come back. + def _OnSubscribe(self, client, userdata, mid, reason_code_list:List[mqtt.ReasonCode], properties): + # We only want to listen for the result of the report subscribe. + if self.ReportSubscribeMid is not None and self.ReportSubscribeMid == mid: + # Ensure the sub was successful. + for r in reason_code_list: + if r.is_failure: + # On any failure, report it and disconnect. + self.Logger.error(f"Sub response for the report subscription reports failure. {r}") + self.Client.disconnect() + return + # At this point, we know the connection was successful, the access code is correct, and the SN is correct. + self.ConsecutivelyFailedConnectionAttempts = 0 - def OnDisconnect(self, client, userdata, disconnect_flags, reason_code, properties): - self.Logger.warn("MQTT disconnected") + # Sub success! Force a full state sync. + self._ForceStateSyncAsync() - def OnMessage(self, client, userdata, msg): + # Fired when there's an incoming MQTT message. + def _OnMessage(self, client, userdata, msg:mqtt.MQTTMessage): try: - doc = json.loads(msg.payload) - self.Logger.info("Bambu "+json.dumps(doc, indent=3)) - if doc is None: - return - self.client.publish(f"device/{self.PrinterSn}/request", '{{"pushing": {{ "sequence_id": 1, "command": "pushall"}}, "user_id":"1234567890"}}') - except Exception: - pass + # Try to deserialize the message. + msg = json.loads(msg.payload) + if msg is None: + raise Exception("Parsed json MQTT message returned None") + + # Print for debugging if desired. + if self.Logger.isEnabledFor(logging.DEBUG): + self.Logger.debug("Incoming Bambu Message:\r\n"+json.dumps(msg, indent=3)) + + # Since we keep a track of the state locally from the partial updates, we need to feed all updates to our state object. + isFirstFullSyncResponse = False + if "print" in msg: + printMsg = msg["print"] + try: + if self.State is None: + # Build the object before we set it. + s = BambuState() + s.OnUpdate(printMsg) + self.State = s + else: + self.State.OnUpdate(printMsg) + except Exception as e: + Sentry.Exception("Exception calling BambuState.OnUpdate", e) + + # Try to detect if this is the response to the first full sync request. + if self.HasDoneFirstFullStateSync is False: + # First make sure the command is the push status. + cmd = printMsg.get("command", None) + if cmd is not None and cmd == "push_status": + # We dont have a 100% great way to know if this is a fully sync message. + # For now, we use this stat. The message we get from a P1P has 59 members in the root, so we use 40 as mark. + if len(printMsg) > 40: + isFirstFullSyncResponse = True + self.HasDoneFirstFullStateSync = True + + # Update the version info if sent. + if "info" in msg: + try: + if self.Version is None: + # Build the object before we set it. + s = BambuVersion(self.Logger) + s.OnUpdate(msg["info"]) + self.Version = s + else: + self.Version.OnUpdate(msg["info"]) + except Exception as e: + Sentry.Exception("Exception calling BambuVersion.OnUpdate", e) + + # Send all messages to the state translator + # This must happen AFTER we update the State object, so it's current. + # We also pass the State object since we know it's not None + try: + self.StateTranslator.OnMqttMessage(msg, self.State, isFirstFullSyncResponse) + except Exception as e: + Sentry.Exception("Exception calling StateTranslator.OnMqttMessage", e) + + except Exception as e: + self.Logger.warn(f"Failed to handle incoming mqtt message. {e} {msg.payload}") + + + # Publishes a message and blocks until it knows if the message send was successful or not. + def _Publish(self, msg:dict) -> bool: + try: + # Print for debugging if desired. + if self.Logger.isEnabledFor(logging.DEBUG): + self.Logger.debug("Incoming Bambu Message:\r\n"+json.dumps(msg, indent=3)) + + # Ensure we are connected. + if self.Client is None or not self.Client.is_connected(): + self.Logger.info("Failed to publish command because we aren't connected.") + return False + + # Try to publish. + state = self.Client.publish(f"device/{self.PrinterSn}/report", json.dumps(msg)) + + # Wait for the message publish to be acked. + # This will throw if the publish fails. + state.wait_for_publish(20) + return True + except Exception as e: + Sentry.Exception("Failed to publish message to bambu printer.", e) + return False + + + # Gets the IP or hostname that should be used for the next connection attempt. + def _GetIpOrHostnameToTry(self) -> str: + # Increment and reset if it's too high. + self.ConsecutivelyFailedConnectionAttempts += 1 + if self.ConsecutivelyFailedConnectionAttempts > 5: + self.ConsecutivelyFailedConnectionAttempts = 0 + + # On the first few attempts, use the expected IP. + # The first attempt will always be attempt 1, since it's reset to 0 and incremented before connecting + configIpOrHostname = self.Config.GetStr(Config.SectionCompanion, Config.CompanionKeyIpOrHostname, None) + if self.ConsecutivelyFailedConnectionAttempts < 3: + return configIpOrHostname + + # If we fail too many times, try to scan for the printer on the local subnet, the IP could have changed. + # Since we 100% identify the printer by the access token and printer SN, we can try to scan for it, and if we find something, it must be it. + ips = NetworkSearch.ScanForInstances_Bambu(self.Logger, self.AccessToken, self.PrinterSn) + + # If we find a different IP, try it. + if len(ips) == 1: + ip = ips[0] + self.Logger.info(f"We found a possible IP for this instance {ip}, trying it now.") + return ip + + # If we don't find anything, just use the config IP. + return configIpOrHostname + + +# A class returned as the result of all commands. +class BambuCommandResult: + + def __init__(self, result:dict = None, connected:bool = True, timeout:bool = False, otherError:str = None, exception:Exception = None) -> None: + self.Connected = connected + self.Timeout = timeout + self.OtherError = otherError + self.Ex = exception + self.Result = result + + + def HasError(self) -> bool: + return self.Ex is not None or self.OtherError is not None or self.Result is None or self.Connected is False or self.Timeout is True + + + def GetLoggingErrorStr(self) -> str: + if self.Ex is not None: + return str(self.Ex) + if self.OtherError is not None: + return self.OtherError + if self.Connected is False: + return "MQTT not connected." + if self.Timeout: + return "Command timeout." + if self.Result is None: + return "No response." + return "No Error" diff --git a/bambu_octoeverywhere/bambucommandhandler.py b/bambu_octoeverywhere/bambucommandhandler.py index 4c89245..49f0652 100644 --- a/bambu_octoeverywhere/bambucommandhandler.py +++ b/bambu_octoeverywhere/bambucommandhandler.py @@ -1,6 +1,6 @@ -#import json +from octoeverywhere.commandhandler import CommandResponse -from octoeverywhere.commandhandler import CommandHandler, CommandResponse +from .bambuclient import BambuClient # This class implements the Platform Command Handler Interface class BambuCommandHandler: @@ -18,148 +18,131 @@ def __init__(self, logger) -> None: # # See the JobStatusV2 class in the service for the object definition. # + # Returning None will result in the "Printer not connected" state. def GetCurrentJobStatus(self): - # result = MoonrakerClient.Get().SendJsonRpcRequest("printer.objects.query", - # { - # "objects": { - # "print_stats": None, # Needed for many things, including GetPrintTimeRemainingEstimateInSeconds_WithPrintStatsAndVirtualSdCardResult - # "gcode_move": None, # Needed for GetPrintTimeRemainingEstimateInSeconds_WithPrintStatsAndVirtualSdCardResult to get the current speed - # "virtual_sdcard": None, # Needed for many things, including GetPrintTimeRemainingEstimateInSeconds_WithPrintStatsAndVirtualSdCardResult - # "extruder": None, # Needed for temps - # "heater_bed": None, # Needed for temps - # # "webhooks": None, - # # "extruder": None, - # # "bed_mesh": None, - # } - # }) - # # Validate - # if result.HasError(): - # self.Logger.error("MoonrakerCommandHandler failed GetCurrentJobStatus() query. "+result.GetLoggingErrorStr()) - # return None - - # # Get the result. - # res = result.GetResult() - - # # Map the state - # state = "idle" - # if "status" in res and "print_stats" in res["status"] and "state" in res["status"]["print_stats"]: - # # https://moonraker.readthedocs.io/en/latest/printer_objects/#print_stats - # mrState = res["status"]["print_stats"]["state"] - # if mrState == "standby": - # state = "idle" - # elif mrState == "printing": - # # This is a special case, we consider "warmingup" a subset of printing. - # if MoonrakerClient.Get().GetMoonrakerCompat().CheckIfPrinterIsWarmingUp_WithPrintStats(result): - # state = "warmingup" - # else: - # state = "printing" - # elif mrState == "paused": - # state = "paused" - # elif mrState == "complete": - # state = "complete" - # elif mrState == "cancelled": - # state = "cancelled" - # elif mrState == "error": - # state = "error" - # else: - # self.Logger.warn("Unknown mrState returned from print_stats: "+str(mrState)) - # else: - # self.Logger.warn("MoonrakerCommandHandler failed to find the print_stats.status") - - # # TODO - If in an error state, set some context as to why. - # errorStr_CanBeNone = None - - # # Get current layer info - # # None = The platform doesn't provide it. - # # 0 = The platform provider it, but there's no info yet. - # # # = The values - # # Note this is similar to how we also do it for notifications. - # currentLayerInt = None - # totalLayersInt = None - # currentLayerRaw, totalLayersRaw = MoonrakerClient.Get().GetMoonrakerCompat().GetCurrentLayerInfo() - # if totalLayersRaw is not None and totalLayersRaw > 0 and currentLayerRaw is not None and currentLayerRaw >= 0: - # currentLayerInt = int(currentLayerRaw) - # totalLayersInt = int(totalLayersRaw) - - # # Get duration and filename. - # durationSec = 0 - # fileName = "" - # if "status" in res and "print_stats" in res["status"]: - # ps = res["status"]["print_stats"] - # # We choose to use print_duration over "total_duration" so we only show the time actually spent printing. This is consistent across platforms. - # if "print_duration" in ps: - # durationSec = int(ps["print_duration"]) - # if "filename" in ps: - # fileName = ps["filename"] - - # # If we have a file name, try to get the current filament usage. - # filamentUsageMm = 0 + # Try to get the current state. + bambuState = BambuClient.Get().GetState() + + # If the state is None, we are disconnected. + if bambuState is None: + # Returning None will be a "connection lost" state. + return None + + # Map the state + # TODO - Add "error" if possible + # Possible states: https://github.com/greghesp/ha-bambulab/blob/e72e343acd3279c9bccba510f94bf0e291fe5aaa/custom_components/bambu_lab/pybambu/const.py#L83C1-L83C21 + state = "idle" + if bambuState.gcode_state is not None: + gcodeState = bambuState.gcode_state + if gcodeState == "IDLE" or gcodeState == "INIT" or gcodeState == "OFFLINE" or gcodeState == "UNKNOWN": + state = "idle" + elif gcodeState == "RUNNING" or gcodeState == "SLICING": + # Only check stg_cur in the known printing state, because sometimes it doesn't get reset to idle when transitioning to an error. + stg = bambuState.stg_cur + # Here's a full list: https://github.com/davglass/bambu-cli/blob/398c24057c71fc6bcc5dbd818bdcacc20833f61c/lib/const.js#L104 + # stg==255 is used as a kind of intenum unknown state when the print is first starting and finishing. + # We can't really use it because it can happen at different points in time and it's not clear what the real state is. + if stg == 2 or stg == 7: + state = "warmingup" + elif stg == 14: + state = "cleaningnozzle" + elif stg == 1: + state = "autobedlevel" + else: + state = "printing" + elif gcodeState == "PAUSE": + state = "paused" + elif gcodeState == "FINISH": + state = "complete" + elif gcodeState == "FAILED": + state = "cancelled" + elif gcodeState == "PREPARE": + state = "warmingup" + else: + self.Logger.warn(f"Unknown gcode_state state in print state: {gcodeState}") + + # TODO - If in an error state, set some context as to why. + errorStr_CanBeNone = None + + # Get current layer info + # None = The platform doesn't provide it. + # 0 = The platform provider it, but there's no info yet. + # # = The values + currentLayerInt = None + totalLayersInt = None + if bambuState.layer_num is not None: + currentLayerInt = int(bambuState.layer_num) + if bambuState.total_layer_num is not None: + totalLayersInt = int(bambuState.total_layer_num) + + # Get duration and filename. + durationSec = 0 + fileName = "" + if bambuState.gcode_file is not None: + fileName = bambuState.gcode_file + #if "gcode_file" in res: + #durationSec = res["gcode_file"] + + # If we have a file name, try to get the current filament usage. + filamentUsageMm = 0 # if fileName is not None and len(fileName) > 0: # filamentUsageMm = FileMetadataCache.Get().GetEstimatedFilamentUsageMm(fileName) - # # Get the progress - # progress = 0.0 - # if "status" in res and "virtual_sdcard" in res["status"]: - # vs = res["status"]["virtual_sdcard"] - # if "progress" in vs: - # # Convert progress 0->1 to 0->100 - # progress = vs["progress"] * 100.0 - - # # Time left can be hard to compute correctly, so use the common function to do it based - # # on what we can get as a best effort. - # timeLeftSec = MoonrakerClient.Get().GetMoonrakerCompat().GetPrintTimeRemainingEstimateInSeconds_WithPrintStatsVirtualSdCardAndGcodeMoveResult(result) - - # # Get the current temps if possible. - # hotendActual = 0.0 - # hotendTarget = 0.0 - # bedTarget = 0.0 - # bedActual = 0.0 - # if "status" in res and "extruder" in res["status"]: - # extruder = res["status"]["extruder"] - # if "temperature" in extruder: - # hotendActual = round(float(extruder["temperature"]), 2) - # if "target" in extruder: - # hotendTarget = round(float(extruder["target"]), 2) - # if "status" in res and "heater_bed" in res["status"]: - # heater_bed = res["status"]["heater_bed"] - # if "temperature" in heater_bed: - # bedActual = round(float(heater_bed["temperature"]), 2) - # if "target" in heater_bed: - # bedTarget = round(float(heater_bed["target"]), 2) + # Get the progress + progress = 0.0 + if bambuState.mc_percent is not None: + progress = float(bambuState.mc_percent) + + # We have special logic to handle the time left count down, since bambu only gives us minutes + # and we want seconds. We can estimate it pretty well by counting down from the last time it changed. + timeLeftSec = bambuState.GetContinuousTimeRemainingSec() + if timeLeftSec is None: + timeLeftSec = 0 + + # Get the current temps if possible. + hotendActual = 0.0 + hotendTarget = 0.0 + bedTarget = 0.0 + bedActual = 0.0 + if bambuState.nozzle_temper is not None: + hotendActual = round(float(bambuState.nozzle_temper), 2) + if bambuState.nozzle_target_temper is not None: + hotendTarget = round(float(bambuState.nozzle_target_temper), 2) + if bambuState.bed_temper is not None: + bedActual = round(float(bambuState.bed_temper), 2) + if bambuState.bed_target_temper is not None: + bedTarget = round(float(bambuState.bed_target_temper), 2) # Build the object and return. return { - "State": "error", - "Error": "bambu", + "State": state, + "Error": errorStr_CanBeNone, + "CurrentPrint": + { + "Progress" : progress, + "DurationSec" : durationSec, + "TimeLeftSec" : timeLeftSec, + "FileName" : fileName, + "EstTotalFilUsedMm" : filamentUsageMm, + "CurrentLayer": currentLayerInt, + "TotalLayers": totalLayersInt, + "Temps": { + "BedActual": bedActual, + "BedTarget": bedTarget, + "HotendActual": hotendActual, + "HotendTarget": hotendTarget, + } + } } - # return { - # "State": state, - # "Error": errorStr_CanBeNone, - # "CurrentPrint": - # { - # "Progress" : progress, - # "DurationSec" : durationSec, - # "TimeLeftSec" : timeLeftSec, - # "FileName" : fileName, - # "EstTotalFilUsedMm" : filamentUsageMm, - # "CurrentLayer": currentLayerInt, - # "TotalLayers": totalLayersInt, - # "Temps": { - # "BedActual": bedActual, - # "BedTarget": bedTarget, - # "HotendActual": hotendActual, - # "HotendTarget": hotendTarget, - # } - # } - # } - # !! Platform Command Handler Interface Function !! # This must return the platform version as a string. def GetPlatformVersionStr(self): - # We don't supply this for moonraker at the moment. - return "1.0.0" + version = BambuClient.Get().GetVersion() + if version is None: + return "0.0.0" + return f"{version.SoftwareVersion}-{version.PrinterName}" # !! Platform Command Handler Interface Function !! @@ -167,14 +150,10 @@ def GetPlatformVersionStr(self): # If not, it must return the correct two error codes accordingly. # This must return a CommandResponse. def ExecutePause(self, smartPause, suppressNotificationBool, disableHotendBool, disableBedBool, zLiftMm, retractFilamentMm, showSmartPausePopup) -> CommandResponse: - # Check the state and that we have a connection to the host. - result = self._CheckIfConnectedAndForExpectedStates(["printing"]) - if result is not None: - return result - - # The smart pause logic handles all pause commands. - #return SmartPause.Get().ExecuteSmartPause(suppressNotificationBool) - return CommandResponse.Success(None) + if BambuClient.Get().SendPause(): + return CommandResponse.Success(None) + else: + return CommandResponse.Error(400, "Failed to send command to printer.") # !! Platform Command Handler Interface Function !! @@ -182,23 +161,10 @@ def ExecutePause(self, smartPause, suppressNotificationBool, disableHotendBool, # If not, it must return the correct two error codes accordingly. # This must return a CommandResponse. def ExecuteResume(self) -> CommandResponse: - # Check the state and that we have a connection to the host. - result = self._CheckIfConnectedAndForExpectedStates(["paused"]) - if result is not None: - return result - - # # Do the resume. - # result = MoonrakerClient.Get().SendJsonRpcRequest("printer.print.resume", {}) - # if result.HasError(): - # self.Logger.error("ExecuteResume failed to request resume. "+result.GetLoggingErrorStr()) - # return CommandResponse.Error(400, "Failed to request resume") - - # # Check the response - # if result.GetResult() != "ok": - # self.Logger.error("ExecuteResume got an invalid request response. "+json.dumps(result.GetResult())) - # return CommandResponse.Error(400, "Invalid request response.") - - return CommandResponse.Success(None) + if BambuClient.Get().SendResume(): + return CommandResponse.Success(None) + else: + return CommandResponse.Error(400, "Failed to send command to printer.") # !! Platform Command Handler Interface Function !! @@ -206,49 +172,7 @@ def ExecuteResume(self) -> CommandResponse: # If not, it must return the correct two error codes accordingly. # This must return a CommandResponse. def ExecuteCancel(self) -> CommandResponse: - # Check the state and that we have a connection to the host. - result = self._CheckIfConnectedAndForExpectedStates(["printing","paused"]) - if result is not None: - return result - - # Do the resume. - # result = MoonrakerClient.Get().SendJsonRpcRequest("printer.print.cancel", {}) - # if result.HasError(): - # self.Logger.error("ExecuteCancel failed to request cancel. "+result.GetLoggingErrorStr()) - # return CommandResponse.Error(400, "Failed to request cancel") - - # # Check the response - # if result.GetResult() != "ok": - # self.Logger.error("ExecuteCancel got an invalid request response. "+json.dumps(result.GetResult())) - # return CommandResponse.Error(400, "Invalid request response.") - - return CommandResponse.Success(None) - - - # Checks if the printer is connected and in the correct state (or states) - # If everything checks out, returns None. Otherwise it returns a CommandResponse - def _CheckIfConnectedAndForExpectedStates(self, stateArray) -> CommandResponse: - # Only allow the pause if the print state is printing, otherwise the system seems to get confused. - # result = MoonrakerClient.Get().SendJsonRpcRequest("printer.objects.query", - # { - # "objects": { - # "print_stats": None - # } - # }) - # if result.HasError(): - # if result.ErrorCode == JsonRpcResponse.OE_ERROR_WS_NOT_CONNECTED: - # self.Logger.error("Command failed because the printer is no connected. "+result.GetLoggingErrorStr()) - # return CommandResponse.Error(CommandHandler.c_CommandError_HostNotConnected, "Printer Not Connected") - # self.Logger.error("Command failed to get state. "+result.GetLoggingErrorStr()) - # return CommandResponse.Error(500, "Error Getting State") - # res = result.GetResult() - # if "status" not in res or "print_stats" not in res["status"] or "state" not in res["status"]["print_stats"]: - # self.Logger.error("Command failed to get state, state not found in dict.") - # return CommandResponse.Error(500, "Error Getting State From Dict") - # state = res["status"]["print_stats"]["state"] - # for s in stateArray: - # if s == state: - # return None - - # self.Logger.warn("Command failed, printer "+state+" not the expected states.") - return CommandResponse.Error(CommandHandler.c_CommandError_InvalidPrinterState, "Wrong State") + if BambuClient.Get().SendCancel(): + return CommandResponse.Success(None) + else: + return CommandResponse.Error(400, "Failed to send command to printer.") diff --git a/bambu_octoeverywhere/bambuhost.py b/bambu_octoeverywhere/bambuhost.py index 3642d3d..fa07ecf 100644 --- a/bambu_octoeverywhere/bambuhost.py +++ b/bambu_octoeverywhere/bambuhost.py @@ -9,6 +9,8 @@ from octoeverywhere.octopingpong import OctoPingPong from octoeverywhere.commandhandler import CommandHandler from octoeverywhere.octoeverywhereimpl import OctoEverywhere +from octoeverywhere.notificationshandler import NotificationsHandler +from octoeverywhere.octohttprequest import OctoHttpRequest from octoeverywhere.Proto.ServerHost import ServerHost from octoeverywhere.compat import Compat @@ -18,8 +20,10 @@ from linux_host.logger import LoggerInit from .bambuclient import BambuClient -from .bambucommandhandler import BambuCommandHandler from .bambuwebcamhelper import BambuWebcamHelper +from .bambucommandhandler import BambuCommandHandler +from .bambustatetranslater import BambuStateTranslator +from .quickcam import QuickCam # This file is the main host for the bambu service. class BambuHost: @@ -27,6 +31,7 @@ class BambuHost: def __init__(self, configDir:str, logDir:str, devConfig_CanBeNone) -> None: # When we create our class, make sure all of our core requirements are created. self.Secrets = None + self.NotificationHandler:NotificationsHandler = None # Let the compat system know this is an Bambu host. Compat.SetIsBambu(True) @@ -64,7 +69,6 @@ def RunBlocking(self, configPath, localStorageDir, repoRoot, devConfig_CanBeNone self.Logger.info("Plugin Version: %s", pluginVersionStr) # Before the first time setup, we must also init the Secrets class and do the migration for the printer id and private key, if needed. - # As of 8/15/2023, we don't store any sensitive things in teh config file, since all config files are sometimes backed up publicly. self.Secrets = Secrets(self.Logger, localStorageDir) # Now, detect if this is a new instance and we need to init our global vars. If so, the setup script will be waiting on this. @@ -87,40 +91,36 @@ def RunBlocking(self, configPath, localStorageDir, repoRoot, devConfig_CanBeNone # Init the mdns client MDns.Init(self.Logger, localStorageDir) - - - # # Setup the http requester. We default to port 80 and assume the frontend can be found there. - # # TODO - parse nginx to see what front ends exist and make them switchable - # # TODO - detect HTTPS port if 80 is not bound. - # frontendPort = self.Config.GetInt(Config.RelaySection, Config.RelayFrontEndPortKey, 80) - # self.Logger.info("Setting up relay with frontend port %s", str(frontendPort)) - # OctoHttpRequest.SetLocalHttpProxyPort(frontendPort) - # OctoHttpRequest.SetLocalHttpProxyIsHttps(False) - # OctoHttpRequest.SetLocalOctoPrintPort(frontendPort) + # For bambu, there's no frontend to connect to, so we disable the http relay system. + OctoHttpRequest.SetDisableHttpRelay(True) # Init the ping pong helper. OctoPingPong.Init(self.Logger, localStorageDir, printerId) if DevLocalServerAddress_CanBeNone is not None: OctoPingPong.Get().DisablePrimaryOverride() - # Setup the snapshot helper - # TODO - webcamHelper = BambuWebcamHelper(self.Logger) + # Setup the webcam helper and QuickCam + QuickCam.Init(self.Logger, self.Config) + webcamHelper = BambuWebcamHelper(self.Logger, self.Config) WebcamHelper.Init(self.Logger, webcamHelper, localStorageDir) + # Setup the state translator and notification handler + stateTranslator = BambuStateTranslator(self.Logger) + self.NotificationHandler = NotificationsHandler(self.Logger, stateTranslator) + self.NotificationHandler.SetPrinterId(printerId) + stateTranslator.SetNotificationHandler(self.NotificationHandler) + # Setup the command handler - # TODO - Notification handler - CommandHandler.Init(self.Logger, None, BambuCommandHandler(self.Logger)) + CommandHandler.Init(self.Logger, self.NotificationHandler, BambuCommandHandler(self.Logger)) - c = BambuClient(self.Logger, self.Config) - c.DoesNothing() + # Setup and start the Bambu Client + BambuClient.Init(self.Logger, self.Config, stateTranslator) # Now start the main runner! OctoEverywhereWsUri = HostCommon.c_OctoEverywhereOctoClientWsUri if DevLocalServerAddress_CanBeNone is not None: OctoEverywhereWsUri = "ws://"+DevLocalServerAddress_CanBeNone+"/octoclientws" - # TODO update types - oe = OctoEverywhere(OctoEverywhereWsUri, printerId, privateKey, self.Logger, self, self, pluginVersionStr, ServerHost.Moonraker, False) + oe = OctoEverywhere(OctoEverywhereWsUri, printerId, privateKey, self.Logger, self, self, pluginVersionStr, ServerHost.Bambu, False) oe.RunBlocking() except Exception as e: Sentry.Exception("!! Exception thrown out of main host run function.", e) @@ -208,6 +208,9 @@ def ShowUiPopup(self, title:str, text:str, msgType:str, actionText:str, actionLi def OnPrimaryConnectionEstablished(self, octoKey, connectedAccounts): self.Logger.info("Primary Connection To OctoEverywhere Established - We Are Ready To Go!") + # Give the octoKey to who needs it. + self.NotificationHandler.SetOctoKey(octoKey) + # Check if this printer is unlinked, if so add a message to the log to help the user setup the printer if desired. # This would be if the skipped the printer link or missed it in the setup script. if connectedAccounts is None or len(connectedAccounts) == 0: diff --git a/bambu_octoeverywhere/bambumodels.py b/bambu_octoeverywhere/bambumodels.py new file mode 100644 index 0000000..9c7cc46 --- /dev/null +++ b/bambu_octoeverywhere/bambumodels.py @@ -0,0 +1,158 @@ +import time +import logging +from enum import Enum + +# Since MQTT syncs a full state and then sends partial updates, we keep track of the full state +# and then apply updates on top of it. We basically keep a locally cached version of the state around. +class BambuState: + + def __init__(self) -> None: + # We only parse out what we currently use. + # We use the same naming as the json in the msg + self.stg_cur:int = None + self.gcode_state:str = None + self.layer_num:int = None + self.total_layer_num:int = None + self.gcode_file:str = None + self.mc_percent:int = None + self.nozzle_temper:int = None + self.nozzle_target_temper:int = None + self.bed_temper:int = None + self.bed_target_temper:int = None + self.mc_remaining_time:int = None + # Custom fields + self.LastTimeRemainingWallClock:float = None + + + # Called when there's a new print message from the printer. + def OnUpdate(self, msg:dict) -> None: + # Get a new value or keep the current. + # Remember that most of these are partial updates and will only have some values. + self.stg_cur = msg.get("stg_cur", self.stg_cur) + self.gcode_state = msg.get("gcode_state", self.gcode_state) + self.layer_num = msg.get("layer_num", self.layer_num) + self.total_layer_num = msg.get("total_layer_num", self.total_layer_num) + self.gcode_file = msg.get("gcode_file", self.gcode_file) + self.mc_percent = msg.get("mc_percent", self.mc_percent) + self.nozzle_temper = msg.get("nozzle_temper", self.nozzle_temper) + self.nozzle_target_temper = msg.get("nozzle_target_temper", self.nozzle_target_temper) + self.bed_temper = msg.get("bed_temper", self.bed_temper) + self.bed_target_temper = msg.get("bed_target_temper", self.bed_target_temper) + + # Time remaining has some custom logic, so as it's queried each time it keep counting down in seconds, since Bambu only gives us minutes. + old_mc_remaining_time = self.mc_remaining_time + self.mc_remaining_time = msg.get("mc_remaining_time", self.mc_remaining_time) + if old_mc_remaining_time != self.mc_remaining_time: + self.LastTimeRemainingWallClock = time.time() + + + # Returns a time reaming value that counts down in seconds, not just minutes. + # Returns null if the time is unknown. + def GetContinuousTimeRemainingSec(self) -> int: + if self.mc_remaining_time is None or self.LastTimeRemainingWallClock is None: + return None + # The slicer holds a constant time while in preparing, so we don't want to fake our countdown either. + if self.gcode_state == "SLICING" or self.gcode_state == "PREPARE": + # Reset the last wall clock time to now, so when we transition to running, we don't snap to a strange offset. + self.LastTimeRemainingWallClock = time.time() + return self.mc_remaining_time * 60.0 + # Compute the time based on when the value last updated. + return int(max(0, (self.mc_remaining_time * 60) - (time.time() - self.LastTimeRemainingWallClock))) + + + # Since there's a lot to consider to figure out if a print is running, this one function acts as common logic across the plugin. + def IsPrinting(self, includePausedAsPrinting:bool) -> bool: + if self.gcode_state is None: + return False + if self.gcode_state == "PAUSE" and includePausedAsPrinting: + return True + # Do we need to consider some of the stg_cur states? + return self.gcode_state == "RUNNING" or self.gcode_state == "SLICING" or self.gcode_state == "PREPARE" + + + # This one function acts as common logic across the plugin. + def IsPaused(self) -> bool: + if self.gcode_state is None: + return False + return self.gcode_state == "PAUSE" + + +# Different types of hardware. +class BambuPrinters(Enum): + Unknown = 1 + X1C = 2 + X1E = 3 + P1P = 10 + P1S = 11 + A1 = 20 + A1Mini = 21 + + +class BambuCPUs(Enum): + Unknown = 1 + ESP32 = 2 # Lower powered CPU used on the A1 and P1P + RV1126= 3 # High powered CPU used on the X1 line + + +# Tracks the version info. +class BambuVersion: + + def __init__(self, logger:logging.Logger) -> None: + self.Logger = logger + # We only parse out what we currently use. + self.SoftwareVersion:str = None + self.HardwareVersion:str = None + self.SerialNumber:str = None + self.ProjectName:str = None + self.Cpu:BambuCPUs = None + self.PrinterName:BambuPrinters = None + + + # Called when there's a new print message from the printer. + def OnUpdate(self, msg:dict) -> None: + module = msg.get("module", None) + if module is None: + return + for m in module: + name = m.get("name", None) + if name is None: + continue + if name == "ota": + self.SoftwareVersion = m.get("sw_ver", self.SoftwareVersion) + elif name == "mc": + self.SerialNumber = m.get("sn", self.SerialNumber) + elif name == "esp32": + self.HardwareVersion = m.get("hw_ver", self.HardwareVersion) + self.ProjectName = m.get("project_name", self.ProjectName) + self.Cpu = BambuCPUs.ESP32 + elif name == "rv1126": + self.HardwareVersion = m.get("hw_ver", self.HardwareVersion) + self.ProjectName = m.get("project_name", self.ProjectName) + self.Cpu = BambuCPUs.RV1126 + + # If we didn't find a hardware, it's unknown. + if self.Cpu is None: + self.Cpu = BambuCPUs.Unknown + + # Now that we have info, map the printer type. + if self.Cpu is not BambuCPUs.Unknown and self.HardwareVersion is not None and self.ProjectName is not None: + if self.Cpu is BambuCPUs.RV1126: + if self.HardwareVersion == "AP05": + self.PrinterName = BambuPrinters.X1C + elif self.HardwareVersion == "AP02": + self.PrinterName = BambuPrinters.X1E + if self.Cpu is BambuCPUs.ESP32: + if self.HardwareVersion == "AP04": + if self.ProjectName == "C11": + self.PrinterName = BambuPrinters.P1P + if self.ProjectName == "C12": + self.PrinterName = BambuPrinters.P1S + if self.HardwareVersion == "AP05": + if self.ProjectName == "N1": + self.PrinterName = BambuPrinters.A1Mini + if self.ProjectName == "N2S": + self.PrinterName = BambuPrinters.A1 + + if self.PrinterName is None or self.PrinterName is BambuPrinters.Unknown: + self.Logger.warn(f"Unknown printer type. CPU:{self.Cpu}, Project Name: {self.ProjectName}, Hardware Version: {self.HardwareVersion}") + self.PrinterName = BambuPrinters.Unknown diff --git a/bambu_octoeverywhere/bambustatetranslater.py b/bambu_octoeverywhere/bambustatetranslater.py new file mode 100644 index 0000000..475095f --- /dev/null +++ b/bambu_octoeverywhere/bambustatetranslater.py @@ -0,0 +1,261 @@ +from octoeverywhere.notificationshandler import NotificationsHandler + +from .bambuclient import BambuClient +from .bambumodels import BambuState + +# This class is responsible for listening to the mqtt messages to fire off notifications +# and to act as the printer state interface for Bambu printers. +class BambuStateTranslator: + + def __init__(self, logger) -> None: + self.Logger = logger + self.NotificationsHandler:NotificationsHandler = None + self.HasPendingPrintStart = False + self.HasPendingPrintPause = False + self.HasPendingPrintResume = False + self.HasPendingPrintFailed = False + self.WasInRunningStateLastUpdate = False + + + def SetNotificationHandler(self, notificationHandler:NotificationsHandler): + self.NotificationsHandler = notificationHandler + + + # Fired when any mqtt message comes in. + # State will always be NOT NONE, since it's going to be created before this call. + # The isFirstFullSyncResponse flag indicates if this is the first full state sync of a new connection. + def OnMqttMessage(self, msg:dict, bambuState:BambuState, isFirstFullSyncResponse:bool): + + # First, if we have a new connection and we just synced, make sure the notification handler is in sync. + if isFirstFullSyncResponse: + # TODO - Ideally we pass the full print duration. + self.NotificationsHandler.OnRestorePrintIfNeeded(bambuState.IsPrinting(False), bambuState.IsPaused(), bambuState.gcode_file, None) + self.WasInRunningStateLastUpdate = False + + # Remember that each delta could have multiple pieces of needed information in them + # And that we will only get the delta updates once! + + # + # Next, handle any explicit onetime command actions. + if "print" in msg: + if "command" in msg["print"]: + # Note about commands. Since these commands come before the push_status update + # The State object MIGHT NOT BE UPDATED TO THE CORRECT STATE WHEN THEY FIRE. + # For example, the gcode_state will be the last state when project_file fires, because it hasn't updated yet. + # That can be really bad, like for ShouldPrintingTimersBeRunning, which will then return an incorrect value. + # Thus we defer the command action until we see the next push_status come in. + command = msg["print"]["command"] + if command == "project_file": + self.HasPendingPrintStart = True + elif command == "pause": + self.HasPendingPrintPause = True + elif command == "resume": + self.HasPendingPrintResume = True + elif command == "stop": + # This is a stop, I think only user generated? + # TODO - will this fire for other types or errors? + self.HasPendingPrintFailed = True + + # If we have a status update, we know our State should be current, so fire any deferred commands. + elif command == "push_status": + if self.HasPendingPrintStart: + # We have to be really careful with this notification, because it kicks off a lot of things. + # We have to wait until the State is reporting RUNNING before we send it, to ensure things like + # ShouldPrintingTimersBeRunning are in a good state when all of the new things query them. + if bambuState.gcode_state is not None and bambuState.gcode_state == "RUNNING": + self.HasPendingPrintStart = False + self.BambuOnStart(bambuState) + else: + self.Logger.info("Deferring print start until the gcode_state is running...") + + if self.HasPendingPrintPause: + self.HasPendingPrintPause = False + self.BambuOnPause(bambuState) + + if self.HasPendingPrintResume: + self.HasPendingPrintResume = False + self.BambuOnResume(bambuState) + + if self.HasPendingPrintFailed: + self.HasPendingPrintFailed = False + self.BambuOnFailed(bambuState) + + # + # Next - Handle notifications that aren't based off one time events. + # + # These are harder to get right, because the printer will send full state objects sometimes when IDLE or PRINTING. + # Thus if we respond to them, it might not be the correct time. For example, the full sync will always include mc_percent, but we + # don't want to fire BambuOnPrintProgress if we aren't printing. + # + # We only want to consider firing these events if we know this isn't the first time sync from a new connection + # and we are currently tacking a print. + if not isFirstFullSyncResponse and self.NotificationsHandler.IsTrackingPrint(): + # Percentage progress update + if "mc_percent" in msg["print"]: + self.BambuOnPrintProgress(bambuState) + + # Complete is hard, because there's no explicitly one time command for print success. + # We also don't want to rely on IsTrackingPrint, because there's a small window where the state could be updated + # and one of the notification threads could check ShouldPrintingTimersBeRunning, it be False, and stop them. + # So, we keep track of if the state was RUNNING and then goes to FINISHED + if bambuState.gcode_state is not None and bambuState.gcode_state == "FINISH": + if self.WasInRunningStateLastUpdate: + # The last state was running and now it's FINISHED, the print is complete. + self.BambuOnComplete(bambuState) + + # Always update the flag. + self.WasInRunningStateLastUpdate = bambuState.gcode_state is not None and bambuState.gcode_state == "RUNNING" + + + def BambuOnStart(self, bambuState:BambuState): + # We can only get the file name from Bambu. + self.NotificationsHandler.OnStarted(self._GetFileNameOrNone(bambuState), 0, 0) + + + def BambuOnComplete(self, bambuState:BambuState): + # We can only get the file name from Bambu. + self.NotificationsHandler.OnDone(self._GetFileNameOrNone(bambuState), None) + + + def BambuOnPause(self, bambuState:BambuState): + self.NotificationsHandler.OnPaused(self._GetFileNameOrNone(bambuState)) + + + def BambuOnResume(self, bambuState:BambuState): + self.NotificationsHandler.OnResume(self._GetFileNameOrNone(bambuState)) + + + def BambuOnFailed(self, bambuState:BambuState): + # TODO - Right now this is only called by what we think are use requested cancels. + # How can we add this for print stopping errors as well? + self.NotificationsHandler.OnFailed(self._GetFileNameOrNone(bambuState), None, "cancelled") + + + def BambuOnPrintProgress(self, bambuState:BambuState): + # We use the "moonrakerProgressFloat" because it's really means a progress that's + # 100% correct and there's no estimations needed. + self.NotificationsHandler.OnPrintProgress(None, float(bambuState.mc_percent)) + + # TODO - Handlers + # + # # Fired when OctoPrint or the printer hits an error. + # def OnError(self, error): + + # # Fired when the waiting command is received from the printer. + # def OnWaiting(self): + + # # Fired when we get a M600 command from the printer to change the filament + # def OnFilamentChange(self): + + # # Fired when the printer needs user interaction to continue + # def OnUserInteractionNeeded(self): + + + def _GetFileNameOrNone(self, bambuState:BambuState) -> str: + return bambuState.gcode_file + + # + # + # Printer State Interface + # + # + + # ! Interface Function ! The entire interface must change if the function is changed. + # This function will get the estimated time remaining for the current print. + # Returns -1 if the estimate is unknown. + def GetPrintTimeRemainingEstimateInSeconds(self): + # Get the current state. + state = BambuClient.Get().GetState() + if state is None: + return -1 + # We use our special logic function that will return a almost perfect seconds based countdown + # instead of the just minutes based countdown from bambu. + timeRemainingSec = state.GetContinuousTimeRemainingSec() + if timeRemainingSec is None: + return -1 + return timeRemainingSec + + + # ! Interface Function ! The entire interface must change if the function is changed. + # If the printer is warming up, this value would be -1. The First Layer Notification logic depends upon this or GetCurrentLayerInfo! + # Returns the current zoffset if known, otherwise -1. + def GetCurrentZOffset(self): + # This is only used for the first layer logic, but only if GetCurrentLayerInfo fails. + # Since our GetCurrentLayerInfo shouldn't always work, this shouldn't really matter. + # We can't get this value, but since it doesn't really matter, we can estimate it. + (currentLayer, _) = self.GetCurrentLayerInfo() + if currentLayer is None: + return -1 + + # Since the standard layer height is 0.20mm, we just use that for a guess. + return currentLayer * 0.2 + + + # ! Interface Function ! The entire interface must change if the function is changed. + # If this platform DOESN'T support getting the layer info from the system, this returns (None, None) + # If the platform does support it... + # If the current value is unknown, (0,0) is returned. + # If the values are known, (currentLayer(int), totalLayers(int)) is returned. + # Note that total layers will always be > 0, but current layer can be 0! + def GetCurrentLayerInfo(self): + state = BambuClient.Get().GetState() + if state is None: + return (None, None) + # We can get accurate and 100% correct layers from Bambu, awesome! + currentLayer = None + totalLayers = None + if state.layer_num is not None: + currentLayer = int(state.layer_num) + if state.total_layer_num is not None: + totalLayers = int(state.total_layer_num) + return (currentLayer, totalLayers) + + + # ! Interface Function ! The entire interface must change if the function is changed. + # Returns True if the printing timers (notifications and gadget) should be running, which is only the printing state. (not even paused) + # False if the printer state is anything else, which means they should stop. + def ShouldPrintingTimersBeRunning(self): + state = BambuClient.Get().GetState() + if state is None: + return False + + gcodeState = state.gcode_state + if gcodeState is None: + return False + + # See the logic in GetCurrentJobStatus for a full description + # Since we don't know 100% of the states, we will fail open. + # Here's a possible list: https://github.com/greghesp/ha-bambulab/blob/e72e343acd3279c9bccba510f94bf0e291fe5aaa/custom_components/bambu_lab/pybambu/const.py#L83C1-L83C21 + if gcodeState == "IDLE" or gcodeState == "FINISH" or gcodeState == "FAILED": + self.Logger.warn("ShouldPrintingTimersBeRunning is not in a printing state: "+str(gcodeState)) + return False + return True + + + # ! Interface Function ! The entire interface must change if the function is changed. + # If called while the print state is "Printing", returns True if the print is currently in the warm-up phase. Otherwise False + def IsPrintWarmingUp(self): + state = BambuClient.Get().GetState() + if state is None: + return False + + # Check if the print timers should be running + # This will weed out any gcode_states where we know we aren't running. + # We have seen stg_cur not get reset in the past when the state transitions to an error. + if not self.ShouldPrintingTimersBeRunning(): + return False + + gcodeState = state.gcode_state + if gcodeState is not None: + # See the logic in GetCurrentJobStatus for a full description + # Here's a possible list: https://github.com/greghesp/ha-bambulab/blob/e72e343acd3279c9bccba510f94bf0e291fe5aaa/custom_components/bambu_lab/pybambu/const.py#L83C1-L83C21 + if gcodeState == "PREPARE" or gcodeState == "SLICING": + return True + + if state.stg_cur is None: + return False + # See the logic in GetCurrentJobStatus for a full description + # Here's a full list: https://github.com/davglass/bambu-cli/blob/398c24057c71fc6bcc5dbd818bdcacc20833f61c/lib/const.js#L104 + if state.stg_cur == 1 or state.stg_cur == 2 or state.stg_cur == 7 or state.stg_cur == 9 or state.stg_cur == 11 or state.stg_cur == 14: + return True + return False diff --git a/bambu_octoeverywhere/bambuwebcamhelper.py b/bambu_octoeverywhere/bambuwebcamhelper.py index 456db44..55a4183 100644 --- a/bambu_octoeverywhere/bambuwebcamhelper.py +++ b/bambu_octoeverywhere/bambuwebcamhelper.py @@ -1,14 +1,27 @@ import logging +import time +import threading +from octoeverywhere.webcamhelper import WebcamSettingItem +from octoeverywhere.octohttprequest import OctoHttpRequest + +from linux_host.config import Config + +from .quickcam import QuickCam -#from octoeverywhere.sentry import Sentry -from octoeverywhere.webcamhelper import WebcamSettingItem#, WebcamHelper # This class implements the webcam platform helper interface for bambu. class BambuWebcamHelper(): - def __init__(self, logger:logging.Logger) -> None: + # These don't really matter, but we define them to keep them consistent + c_SpecialMockSnapshotPath = "bambu-special-snapshot" + c_SpecialMockStreamPath = "bambu-special-stream" + c_OeStreamBoundaryString = "oestreamboundary" + + + def __init__(self, logger:logging.Logger, config:Config) -> None: self.Logger = logger + self.Config = config # !! Interface Function !! @@ -17,26 +30,105 @@ def __init__(self, logger:logging.Logger) -> None: # The order the webcams are returned is the order the user will see in any selection UIs. # Returns None on failure. def GetWebcamConfig(self): - return [WebcamSettingItem("Default", "self.SnapshotUrl", "self.StreamUrl", "self.FlipH", "self.FlipV", "self.Rotation")] - - # # Kick the settings worker since the webcam was accessed. - # self.KickOffWebcamSettingsUpdate() - - # # Grab the lock to see what we should be returning. - # with self.ResultsLock: - # # If auto settings are enabled, return any cached auto settings that we found. - # # If we have anything, make a copy of the array and return it. - # if self.EnableAutoSettings: - # if len(self.AutoSettingsResults) != 0: - # results = [] - # for i in self.AutoSettingsResults: - # results.append(i) - # return results - # # If we don't have auto settings enabled or we don't have any results, return what we have in memory. - # # This will either be the default values or a values that the user has set. - # item = WebcamSettingItem("Default", self.SnapshotUrl, self.StreamUrl, self.FlipH, self.FlipV, self.Rotation) - # # Validate the settings, but always return them. - # item.Validate(self.Logger) - # return [ - # item - # ] + # Bambu has a special webcam setup where there's only one cam and we need to get in a special way, + # So we return this one default webcam object. + return [WebcamSettingItem("Default", BambuWebcamHelper.c_SpecialMockSnapshotPath, BambuWebcamHelper.c_SpecialMockStreamPath, False, False, 0)] + + + # !! Optional Interface Function !! + # If defined, this function must handle ALL snapshot requests for the platform. + # + # On failure, return None + # On success, this will return a valid OctoHttpRequest that's fully filled out. + # The snapshot will always already be fully read, and will be FullBodyBuffer var. + def GetSnapshot_Override(self, cameraName:str): + # Try to get a snapshot from our QuickCam system. + img = QuickCam.Get().GetCurrentImage() + if img is None: + return None + + # If we get an image, return it! + headers = { + "Content-Type": "image/jpeg" + } + return OctoHttpRequest.Result(200, headers, BambuWebcamHelper.c_SpecialMockSnapshotPath, False, fullBodyBuffer=img) + + + # !! Optional Interface Function !! + # If defined, this function must handle ALL stream requests for the platform. + # + # On failure, return None + # On success, this will return a valid OctoHttpRequest that's fully filled out. + # This must return an OctoHttpRequest object with a custom body read stream. + def GetStream_Override(self, cameraName:str): + # We must create a new instance of this class per stream to ensure all of the vars stay in it's context + # and the streams are cleaned up properly. + sm = StreamInstance(self.Logger) + return sm.StartWebRequest() + + +# Stream Instance is a class that is created per web stream to handle streaming QuickCam images into the http stream. +# It must be created per http request so it can manage it's own local vars. +class StreamInstance: + def __init__(self, logger:logging.Logger) -> None: + self.Logger = logger + self.IsFirstSend = True + self.StreamOpenTimeSec = time.time() + self.ImageReadyEvent = threading.Event() + self.AwaitingImage:bytearray = None + + + def StartWebRequest(self) -> OctoHttpRequest.Result: + # First, try to get a snapshot. This will determine if we are able to get a stream or not. + # If we can't start the stream, then we don't return success. + # We will also use this first image to start the stream, to get it going ASAP. + self.AwaitingImage = QuickCam.Get().GetCurrentImage() + if self.AwaitingImage is None: + return None + + # Note! We must be sure to call DetachImageStreamCallback to remove this stream callback! + QuickCam.Get().AttachImageStreamCallback(self._NewImageCallback) + + # We must set the content type so that the web browser knows what kind of stream to expect. + headers = { + "content-type": f"multipart/x-mixed-replace; boundary={BambuWebcamHelper.c_OeStreamBoundaryString}", + } + # Return a result object with out callbacks setup for the stream body. + return OctoHttpRequest.Result(200, headers, BambuWebcamHelper.c_SpecialMockStreamPath, False, customBodyStreamCallback=self._CustomBodyStreamRead, customBodyStreamClosedCallback=self._CustomBodyStreamClosed) + + + # Define the callback we will get from QuickCam when there's a new image ready for us to send. + def _NewImageCallback(self, imgBuffer:bytearray): + self.AwaitingImage = imgBuffer + self.ImageReadyEvent.set() + + + # Define a callback for our http body reading system to call when it needs data. + def _CustomBodyStreamRead(self) -> bytearray: + while True: + # See if we can capture an image. There might already be a new image we don't even have to wait for. + capturedImage = self.AwaitingImage + if capturedImage is not None: + # If so, clear the awaiting image and reset the event. + self.AwaitingImage = None + self.ImageReadyEvent.clear() + + # Build the buffer to send + header = f"--{BambuWebcamHelper.c_OeStreamBoundaryString}\r\nContent-Type: image/jpeg\r\nContent-Length: {len(capturedImage)}\r\n\r\n" + imageChunkBuffer = header.encode('utf-8') + capturedImage + b"\r\n" + header.encode('utf-8') + capturedImage + b"\r\n" + + # TODO - I don't know why, but chrome seems to delay the rendering of the image until it gets two? + # This could be something in the pipeline not flushing correctly, or other things. But for now, on the first send we double the image to make it render instantly. + if self.IsFirstSend: + imageChunkBuffer = imageChunkBuffer + imageChunkBuffer + self.IsFirstSend = False + self.Logger.info(f"QuickCam took {time.time()-self.StreamOpenTimeSec} seconds from stream open to first image sent.") + return imageChunkBuffer + # If we didn't get an image, wait on the event for a new one. + self.ImageReadyEvent.wait() + + + # Define a callback for when the http stream is closed. + def _CustomBodyStreamClosed(self) -> None: + # It's important this is called so the stream will be detached! + QuickCam.Get().DetachImageStreamCallback(self._NewImageCallback) diff --git a/bambu_octoeverywhere/quickcam.py b/bambu_octoeverywhere/quickcam.py new file mode 100644 index 0000000..6aee57c --- /dev/null +++ b/bambu_octoeverywhere/quickcam.py @@ -0,0 +1,224 @@ +import logging +import threading +import struct +import ssl +import socket +import time + +from octoeverywhere.sentry import Sentry + +from linux_host.config import Config + +from .bambuclient import BambuClient + +# The goal of this class is to handle webcam streaming and snapshots. The idea is since we need to establish a socket and stream to even get snapshots, +# rather than doing it over and over, we will keep the stream alive for a short period of time and take snapshots, so when the user wants them, they are ready. +class QuickCam: + + # The amount of time the capture thread will stay connected before it will close. + # Whenever an image is accessed, the time is reset. + c_CaptureThreadTimeoutSec = 60 + + _Instance = None + + @staticmethod + def Init(logger:logging.Logger, config:Config): + QuickCam._Instance = QuickCam(logger, config) + + + @staticmethod + def Get(): + return QuickCam._Instance + + + def __init__(self, logger:logging.Logger, config:Config ) -> None: + self.Logger = logger + self.Config = config + self.Lock = threading.Lock() + self.ImageReady = threading.Event() + self.IsCaptureThreadRunning = False + self.CurrentImage:bytearray = None + self.LastImageRequestTimeSec:float = 0.0 + self.ImageStreamCallbacks = [] + self.ImageStreamCallbackLock = threading.Lock() + + + # Tries to get the current image from the printer and returns it as a raw jpeg. + # This will return None if it fails. + def GetCurrentImage(self) -> bytearray: + # Set the last time someone requested an image. + self.LastImageRequestTimeSec = time.time() + + # If there is a current image, return it. + img = self.CurrentImage + if img is not None: + return img + + # We will try to kick the thread twice, just incase it was in the middle of cleaning + # up when we called _ensureCaptureThreadRunning the first time. + kickAttempt = 0 + while kickAttempt < 2: + kickAttempt += 1 + self._ensureCaptureThreadRunning() + # For the timeout, we want to make it quite long. The reason is a lot of things depend on this snapshot + # like Gadget, Notification images, the stream capture system, and more. + # Some printers can take a long time to get the socket ready and working, so we want to give them + # a lot of time. It's better to have a longer delay than get no snapshot. + # Since we loop twice, this will be a 8 second delay max. + self.ImageReady.wait(4) + if self.CurrentImage is not None: + return self.CurrentImage + return self.CurrentImage + + + # Used to attach a new stream handler to receive callbacks when an image is ready. + # Note a call to detach must be called as well! + def AttachImageStreamCallback(self, callback): + with self.ImageStreamCallbackLock: + self.ImageStreamCallbacks.append(callback) + + + # Used to detach a new stream handler to receive callbacks when an image is ready. + def DetachImageStreamCallback(self, callback): + with self.ImageStreamCallbackLock: + self.ImageStreamCallbacks.remove(callback) + + + # Called when there's a new image from the capture thread. + def _SetNewImage(self, img:bytearray) -> None: + # Set the new image. + self.CurrentImage = img + # Release anyone waiting on it. + self.ImageReady.set() + # Fire the callbacks, if there are any. + with self.ImageStreamCallbackLock: + if len(self.ImageStreamCallbacks) > 0: + # Update the last image request time to ensure the stream keeps going. + self.LastImageRequestTimeSec = time.time() + for callback in self.ImageStreamCallbacks: + callback(self.CurrentImage) + + + # Call to make sure the capture thread is running. + def _ensureCaptureThreadRunning(self): + with self.Lock: + if self.IsCaptureThreadRunning: + return + self.Logger.info("QuickCam capture thread starting.") + self.IsCaptureThreadRunning = True + t = threading.Thread(target=self._captureThread) + t.daemon = True + t.start() + + + # Does the image image capture work. + def _captureThread(self): + try: + authData = bytearray() + accessCode = self.Config.GetStr(Config.SectionBambu, Config.BambuAccessToken, None) + ipOrHostname = self.Config.GetStr(Config.SectionCompanion, Config.CompanionKeyIpOrHostname, None) + if accessCode is None or ipOrHostname is None: + raise Exception("QuickCam doesn't have a access code or ip to use.") + + # Build the auth packet + authData += struct.pack(" QuickCam.c_CaptureThreadTimeoutSec: + # TODO - For now, we don't stop the webcam loop while the printer is printing. + # This allows for notifications, Gadget, snapshots, streams, and such to load super easily. + # We need to measure the load on this though. + state = BambuClient.Get().GetState() + if state is None or not state.IsPrinting(True): + # This will invoke the finally clause and leave. + return + + # If the expected image size is 0, then this is the first read of 16 bytes for the header. + if expectedImageSize == 0: + if len(data) != 16: + raise Exception("QuickCam capture thread got a first payload that was longer than 16.") + expectedImageSize = int.from_bytes(data[0:3], byteorder='little') + # Otherwise, we are building an image + else: + # Always add to the current buffer. + imgBuffer += data + + # Check if the image is done. + if len(imgBuffer) == expectedImageSize: + # We have the full image. Sanity check the jpeg start and end bytes exist. + if imgBuffer[:4] != jpegStartSequence: + raise Exception("QuickCam got an image of the expected size, but we failed to find the jpeg start sequence.") + elif imgBuffer[-2:] != jpegEndSequence: + raise Exception("QuickCam got an image of the expected size, but we failed to find the jpeg end sequence.") + self._SetNewImage(imgBuffer) + expectedImageSize = 0 + imgBuffer = bytearray() + + # Sanity check we didn't get misaligned from the stream. + elif len(imgBuffer) > expectedImageSize: + raise Exception(f"QuickCam was building an image expected to be {expectedImageSize} but ended up with a buffer that was {imgBuffer}") + + except Exception as e: + # We have seen times where random errors are returned, like on boot or if the stream is opened too soon after closing. + # This exception block is designed to eat any connection or buffer parsing errors, eat them, and try again. + self.Logger.warn("Exception in QuickCam capture thread. "+str(e)) + time.sleep(2) + except Exception as e: + Sentry.Exception("Exception in QuickCam capture thread. ", e) + finally: + # Before exit the thread... + # Note that order is important here! + # Ensure we clear the image ready event. + self.ImageReady.clear() + # Clear the flag that we are running. + with self.Lock: + self.IsCaptureThreadRunning = False + # And ensure that the current image is cleaned up, so clients don't get a stale image. + self.CurrentImage = None + self.Logger.info("QuickCam capture thread exit.") diff --git a/developer/runpylint.cmd b/developer/runpylint.cmd index 698eaed..a6d7857 100644 --- a/developer/runpylint.cmd +++ b/developer/runpylint.cmd @@ -1,5 +1,20 @@ @echo off REM This is just for local dev help, the github workflow does the real checkin linting. -pylint ..\octoeverywhere\ -pylint ..\octoprint_octoeverywhere\ -pylint ..\moonraker_octoeverywhere\ \ No newline at end of file +REM we must cd into the root and run, otherwise pylint will not find the pylintrc file. +cd .. +echo Running pylint on octoeverywhere +pylint .\octoeverywhere\ + +echo Running pylint on octoprint_octoeverywhere +pylint .\octoprint_octoeverywhere\ +echo Running pylint on moonraker_octoeverywhere +pylint .\moonraker_octoeverywhere\ +echo Running pylint on bambu_octoeverywhere +pylint .\bambu_octoeverywhere\ + +echo Running pylint on py_installer +pylint .\py_installer\ +echo Running pylint on linux_host +pylint .\linux_host\ + +cd developer \ No newline at end of file diff --git a/developer/runpylint.sh b/developer/runpylint.sh index 1e96df7..d656dae 100755 --- a/developer/runpylint.sh +++ b/developer/runpylint.sh @@ -1,13 +1,25 @@ cd .. + +echo "Ensuring required PY packages are installed..." +pip install pylint octoprint==1.9.0 +pip install -r requirements.txt + +echo "" +echo "" +echo "" echo "Testing OctoEverywhere Module..." pylint ./octoeverywhere/ + echo "Testing OctoPrint Module..." pylint ./octoprint_octoeverywhere/ echo "Testing Moonraker Module..." pylint ./moonraker_octoeverywhere/ -echo "Testing Linux Host Module..." -pylint ./linux_host/ echo "Testing Bambu Module..." pylint ./bambu_octoeverywhere/ + +echo "Testing Linux Host Module..." +pylint ./linux_host/ echo "Testing Moonraker Installer Module..." -pylint ./py_installer/ \ No newline at end of file +pylint ./py_installer/ + +cd developer diff --git a/linux_host/networksearch.py b/linux_host/networksearch.py new file mode 100644 index 0000000..76a4407 --- /dev/null +++ b/linux_host/networksearch.py @@ -0,0 +1,198 @@ +import ssl +import socket +import logging +import threading +from typing import List + +import paho.mqtt.client as mqtt + +class NetworkValidationResult: + def __init__(self, failedToConnect:bool = False, failedAuth:bool = False, failSn:bool = False, exception:Exception = None) -> None: + self.FailedToConnect = failedToConnect + self.FailedAuth = failedAuth + self.FailedSerialNumber = failSn + self.Exception = exception + + + def Success(self) -> bool: + return not self.FailedToConnect and not self.FailedAuth and not self.FailedSerialNumber and self.Exception is None + + +# A helper class that allows for validating and or searching for Moonraker or Bambu printers on the local LAN. +class NetworkSearch: + + # The default port all Bambu printers will run MQTT on. + c_BambuDefaultPortStr = "8883" + + + # Scans the local IP LAN subset for Bambu servers that successfully authorize given the access code and printer sn. + @staticmethod + def ScanForInstances_Bambu(logger:logging.Logger, accessCode:str, printerSn:str, portStr:str = None) -> List[str]: + def callback(ip:str): + return NetworkSearch.ValidateConnection_Bambu(logger, ip, accessCode, printerSn, portStr, timeoutSec=5) + return NetworkSearch._ScanForInstances(logger, callback) + + + # Given the ip, accessCode, printerSn, and optionally port, this will check if the printer is connectable. + # Returns a NetworkValidationResult with the results. + @staticmethod + def ValidateConnection_Bambu(logger:logging.Logger, ipOrHostname:str, accessCode:str, printerSn:str, portStr:str = None, timeoutSec:float = 5.0) -> NetworkValidationResult: + client:mqtt.Client = None + try: + if portStr is None: + portStr = NetworkSearch.c_BambuDefaultPortStr + port = int(portStr) + logger.debug(f"Testing for Bambu on {ipOrHostname}:{port}") + result = {} + result["Event"] = threading.Event() + client = mqtt.Client(mqtt.CallbackAPIVersion.VERSION2, userdata=result) + client.tls_set(tls_version=ssl.PROTOCOL_TLS, cert_reqs=ssl.CERT_NONE) + client.tls_insecure_set(True) + client.username_pw_set("bblp", accessCode) + + def connect(client:mqtt.Client, userdata:dict, flags, reason_code:mqtt.ReasonCode, properties): + # If auth is wrong, we will get a connect callback with a failure "Not authorized" + if reason_code.is_failure: + logger.debug(f"Bambu {ipOrHostname} connection failure: {reason_code}") + client.disconnect() + userdata["Event"].set() + return + + # If the connection was successful, the auth was valid. + logger.debug(f"Bambu {ipOrHostname} connected.") + userdata["IsAuthorized"] = True + + # Try to sub, to make sure the SN is correct. + # For most bambu printers, the socket will disconnect if this fails, it doesn't bother to send the sub failed message. + (result, mid) = client.subscribe(f"device/{printerSn}/report") + if result != mqtt.MQTT_ERR_SUCCESS or mid is None: + logger.debug(f"Bambu {ipOrHostname} failed to send subscribe request.") + client.disconnect() + userdata["Event"].set() + userdata["ReportMid"] = mid + + def disconnect(client, userdata:dict, disconnect_flags, reason_code, properties): + logger.debug(f"Bambu {ipOrHostname} disconnected.") + userdata["Event"].set() + + def subscribe(client, userdata:dict, mid, reason_code_list:List[mqtt.ReasonCode], properties): + if "ReportMid" in userdata and mid == userdata["ReportMid"]: + # If this is the sub report, check the status and disconnect. + failedSn = False + for r in reason_code_list: + if r.is_failure: + # On any failure, report it and disconnect. + logger.debug(f"Bambu {ipOrHostname} Sub response for the report subscription reports failure. {r}") + failedSn = True + if not failedSn: + userdata["SnSubSuccess"] = True + logger.debug(f"Bambu {ipOrHostname} Sub success, the serial number is good") + client.disconnect() + userdata["Event"].set() + + # Setup functions and connect. + client.on_connect = connect + client.on_disconnect = disconnect + client.on_subscribe = subscribe + + # Try to connect, this will throw if it fails to find any server to connect to. + failedToConnect = True + try: + client.connect(ipOrHostname, port, keepalive=60) + failedToConnect = False + client.loop_start() + except Exception as e: + logger.debug(f"Bambu {ipOrHostname} - connection failure {e}") + + # Wait for the timeout. + if not failedToConnect: + result["Event"].wait(timeoutSec) + + # Walk though the connection and see how far we got. + failedAuth = True + failedSn = True + if "IsAuthorized" in result: + failedAuth = False + if "SnSubSuccess" in result: + failedSn = False + + return NetworkValidationResult(failedToConnect, failedAuth, failedSn) + + except Exception as e: + return NetworkValidationResult(exception=e) + finally: + # Ensure we alway clean up. + try: + client.disconnect() + except Exception: + pass + try: + client.loop_stop() + except Exception: + pass + + + # Scans the IP subset for server instances. + # testConFunction must be a function func(ip:str) -> NetworkValidationResult + # Returns a list of IPs that reported Success() == True + @staticmethod + def _ScanForInstances(logger:logging.Logger, testConFunction) -> List[str]: + foundIps = [] + try: + localIp = NetworkSearch._TryToGetLocalIp() + if localIp is None or len(localIp) == 0: + logger.debug("Failed to get local IP") + return foundIps + logger.debug(f"Local IP found as: {localIp}") + if ":" in localIp: + logger.info("IPv6 addresses aren't supported for local discovery.") + return foundIps + lastDot = localIp.rfind(".") + if lastDot == -1: + logger.info("Failed to find last dot in local IP?") + return foundIps + ipPrefix = localIp[:lastDot+1] + + counter = 0 + doneThreads = [0] + totalThreads = 255 + threadLock = threading.Lock() + doneEvent = threading.Event() + while counter <= totalThreads: + fullIp = ipPrefix + str(counter) + def threadFunc(ip): + try: + result = testConFunction(ip) + with threadLock: + if result.Success(): + foundIps.append(ip) + doneThreads[0] += 1 + if doneThreads[0] == totalThreads: + doneEvent.set() + except Exception as e: + logger.error(f"Server scan failed for {ip} "+str(e)) + t = threading.Thread(target=threadFunc, args=[fullIp]) + t.start() + counter += 1 + doneEvent.wait() + return foundIps + except Exception as e: + logger.error("Failed to scan for server instances. "+str(e)) + return foundIps + + + @staticmethod + def _TryToGetLocalIp() -> str: + # Find the local IP. Works on Windows and Linux. Always gets the correct routable IP. + # https://stackoverflow.com/a/28950776 + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + ip = None + try: + # doesn't even have to be reachable + s.connect(('1.1.1.1', 1)) + ip = s.getsockname()[0] + except Exception: + pass + finally: + s.close() + return ip diff --git a/moonraker_octoeverywhere/moonrakerclient.py b/moonraker_octoeverywhere/moonrakerclient.py index b2f9025..24ee0a1 100644 --- a/moonraker_octoeverywhere/moonrakerclient.py +++ b/moonraker_octoeverywhere/moonrakerclient.py @@ -1083,7 +1083,7 @@ def _InitPrintStateForFreshConnect(self): fileName_CanBeNone = stats["filename"] totalDurationFloatSec_CanBeNone = stats["total_duration"] # Use the total duration self.Logger.info("Printer state at socket connect is: "+state) - self.NotificationHandler.OnRestorePrintIfNeeded(state, fileName_CanBeNone, totalDurationFloatSec_CanBeNone) + self.NotificationHandler.OnRestorePrintIfNeeded(state == "printing", state == "paused", fileName_CanBeNone, totalDurationFloatSec_CanBeNone) # Queries moonraker for the current printer stats. diff --git a/octoeverywhere/WebStream/octowebstreamhttphelper.py b/octoeverywhere/WebStream/octowebstreamhttphelper.py index a69b18c..2a75aed 100644 --- a/octoeverywhere/WebStream/octowebstreamhttphelper.py +++ b/octoeverywhere/WebStream/octowebstreamhttphelper.py @@ -46,6 +46,7 @@ def __init__(self, streamId, logger:logging.Logger, webStream, webStreamOpenMsg, self.CompressionTimeSec = -1 self.MissingBoundaryWarningCounter = 0 self.IsUsingFullBodyBuffer = False + self.IsUsingCustomBodyStreamCallbacks = False # If this doesn't not equal None, it means we know how much data to expect. self.KnownFullStreamUploadSizeBytes = None @@ -170,6 +171,13 @@ def executeHttpRequest(self): # This HandleCommand wil return a mock OctoHttpResult, including a full mock response object. octoHttpResult = CommandHandler.Get().HandleCommand(httpInitialContext, self.UploadBuffer) else: + # This is a normal web request, first ensure they are allowed. + if OctoHttpRequest.GetDisableHttpRelay(): + self.Logger.warn("OctoWebStreamHttpHelper got a request but the http relay is disabled.") + self.WebStream.SetClosedDueToFailedRequestConnection() + self.WebStream.Close() + return + # For all web requests, check our in memory read-to-go cache. # If available, this will return the object. On a miss it will return None if Compat.HasSlipstream(): @@ -193,27 +201,19 @@ def executeHttpRequest(self): return # On success, unpack the result. - response = octoHttpResult.Result uri = octoHttpResult.Url requestExecutionEnd = time.time() - # If there's no response, it means we failed to connect to whatever the request was trying to connect to. - # Since the request failed, we want to just close the stream, since it's not a protocol failure. - if response is None: - self.Logger.warn(self.getLogMsgPrefix() + " failed to make http request. There was no response.") - self.WebStream.Close() - return - # Now that we have a valid response, use a with block to ensure no matter what it gets closed when we leave. # This is important since we use the stream flag, otherwise close() will not get called and the connection will remain open. # Note that close() could throw in bad cases, but that's ok because this function is allowed to throw on errors and the octostream will be cleaned up. - with response: + with octoHttpResult: # As a caching technique, if the request has the correct modified headers and the response has them as well, send back a 304, # which indicates the body hasn't been modified and we can save the bandwidth by not sending it. # We need to do this before we process the response headers. # This function will check if we want to do a 304 return and update the request correctly. - self.checkForNotModifiedCacheAndUpdateResponseIfSo(sendHeaders, response) + self.checkForNotModifiedCacheAndUpdateResponseIfSo(sendHeaders, octoHttpResult) # Before we check the headers, check if we are using a full body buffer. @@ -235,13 +235,19 @@ def executeHttpRequest(self): fullContentBufferSize = octoHttpResult.BodyBufferPreCompressSize # See what the current header is (if there is one). If it's set, it should match. - if c_contentLengthHeaderKeyLower in response.headers: - curHeaderLen = int(response.headers[c_contentLengthHeaderKeyLower]) + if c_contentLengthHeaderKeyLower in octoHttpResult.Headers: + curHeaderLen = int(octoHttpResult.Headers[c_contentLengthHeaderKeyLower]) if curHeaderLen != fullContentBufferSize: self.Logger.error(f"Request {uri} had a content length set ({curHeaderLen}) but its different from the full body length size: {fullContentBufferSize}") # Ensure the header is set to the current buffer size. - response.headers[c_contentLengthHeaderKeyLower] = str(fullContentBufferSize) + octoHttpResult.Headers[c_contentLengthHeaderKeyLower] = str(fullContentBufferSize) + + # Next, check if this response is using a custom body callback + if octoHttpResult.GetCustomBodyStreamCallback is not None: + # We set this flag so other parts of this class that need to know if we are using it or not + # this way we only have one check that enables or disables it. + self.IsUsingCustomBodyStreamCallbacks = True # Look at the headers to see what kind of response we are dealing with. @@ -252,14 +258,15 @@ def executeHttpRequest(self): boundaryStr = None # Pull out the content type value, so we can use it to figure out if we want to compress this data or not contentTypeLower =None - for name in response.headers: + headers = octoHttpResult.Headers + for name, value in headers.items(): nameLower = name.lower() if nameLower == c_contentLengthHeaderKeyLower: - contentLength = int(response.headers[name]) + contentLength = int(value) elif nameLower == "content-type": - contentTypeLower = response.headers[name].lower() + contentTypeLower = value.lower() # Look for a boundary string, something like this: `multipart/x-mixed-replace;boundary=boundarydonotcross` indexOfBoundaryStart = contentTypeLower.find('boundary=') @@ -267,7 +274,7 @@ def executeHttpRequest(self): # Move past the string we found indexOfBoundaryStart += len('boundary=') # We should find a boundary, use the original case to parse it out. - boundaryStr = response.headers[name][indexOfBoundaryStart:].strip() + boundaryStr = value[indexOfBoundaryStart:].strip() if len(boundaryStr) == 0: self.Logger.error("We found a boundary stream, but didn't find the boundary string. "+ contentTypeLower) continue @@ -275,7 +282,7 @@ def executeHttpRequest(self): elif nameLower == "location": # We have noticed that some proxy servers aren't setup correctly to forward the x-forwarded-for and such headers. # So when the web server responds back with a 301 or 302, the location header might not have the correct hostname, instead an ip like 127.0.0.1. - response.headers[name] = HeaderHelper.CorrectLocationResponseHeaderIfNeeded(self.Logger, uri, response.headers[name], sendHeaders) + octoHttpResult.Headers[name] = HeaderHelper.CorrectLocationResponseHeaderIfNeeded(self.Logger, uri, value, sendHeaders) # We also look at the content-type to determine if we should add compression to this request or not. # general rule of thumb is that compression is quite cheap but really helps with text, so we should compress when we @@ -284,8 +291,8 @@ def executeHttpRequest(self): # Since streams with unknown content-lengths can run for a while, report now when we start one. # If the status code is 304 or 204, we don't expect content. - if self.Logger.isEnabledFor(logging.DEBUG) and contentLength is None and response.status_code != 304 and response.status_code != 204: - self.Logger.debug(self.getLogMsgPrefix() + "STARTING " + method+" [upload:"+str(format(requestExecutionStart - self.OpenedTime, '.3f'))+"s; request_exe:"+str(format(requestExecutionEnd - requestExecutionStart, '.3f'))+"s; ] type:"+str(contentTypeLower)+" status:"+str(response.status_code)+" for " + uri) + if self.Logger.isEnabledFor(logging.DEBUG) and contentLength is None and octoHttpResult.StatusCode != 304 and octoHttpResult.StatusCode != 204: + self.Logger.debug(self.getLogMsgPrefix() + "STARTING " + method+" [upload:"+str(format(requestExecutionStart - self.OpenedTime, '.3f'))+"s; request_exe:"+str(format(requestExecutionEnd - requestExecutionStart, '.3f'))+"s; ] type:"+str(contentTypeLower)+" status:"+str(octoHttpResult.StatusCode)+" for " + uri) # Check for a response handler and if we have one, check if it might want to edit the response of this call. # If so, it will return a context object. If not, it will return None. @@ -313,7 +320,7 @@ def executeHttpRequest(self): # Unless we are skipping the body read, do it now. # If there's a 304, we might have a body, but we don't want to read it. # If the response is 204, there will be no content, so don't bother. - if response.status_code == 304 or response.status_code == 204: + if octoHttpResult.StatusCode == 304 or octoHttpResult.StatusCode == 204: # Use zero read defaults. nonCompressedBodyReadSize = 0 lastBodyReadLength = 0 @@ -322,7 +329,7 @@ def executeHttpRequest(self): # Start by reading data from the response. # This function will return a read length of 0 and a null data offset if there's nothing to read. # Otherwise, it will return the length of the read data and the data offset in the buffer. - nonCompressedBodyReadSize, lastBodyReadLength, dataOffset = self.readContentFromBodyAndMakeDataVector(builder, octoHttpResult, response, boundaryStr, compressBody, contentTypeLower, contentLength, responseHandlerContext) + nonCompressedBodyReadSize, lastBodyReadLength, dataOffset = self.readContentFromBodyAndMakeDataVector(builder, octoHttpResult, boundaryStr, compressBody, contentTypeLower, contentLength, responseHandlerContext) contentReadBytes += lastBodyReadLength nonCompressedContentReadSizeBytes += nonCompressedBodyReadSize @@ -358,10 +365,10 @@ def executeHttpRequest(self): statusCode = None if isFirstResponse is True: # Set the status code, so it's sent. - statusCode = response.status_code + statusCode = octoHttpResult.StatusCode # Gather the headers, if there are any. This will return None if there are no headers to send. - headerVectorOffset = self.buildHeaderVector(builder, response) + headerVectorOffset = self.buildHeaderVector(builder, octoHttpResult) # Build the initial context. We should always send a http initial context on the first response, # even if there are no headers in t. @@ -437,13 +444,14 @@ def executeHttpRequest(self): # Log about it - only if debug is enabled. Otherwise, we don't want to waste time making the log string. responseWriteDone = time.time() if self.Logger.isEnabledFor(logging.DEBUG): - self.Logger.debug(self.getLogMsgPrefix() + method+" [upload:"+str(format(requestExecutionStart - self.OpenedTime, '.3f'))+"s; request_exe:"+str(format(requestExecutionEnd - requestExecutionStart, '.3f'))+"s; send:"+str(format(responseWriteDone - requestExecutionEnd, '.3f'))+"s; body_read:"+str(format(self.BodyReadTimeSec, '.3f'))+"s; compress:"+str(format(self.CompressionTimeSec, '.3f'))+"s; octo_stream_upload:"+str(format(self.ServiceUploadTimeSec, '.3f'))+"s] size:("+str(nonCompressedContentReadSizeBytes)+"->"+str(contentReadBytes)+") compressed:"+str(compressBody)+" msgcount:"+str(messageCount)+" microreads:"+str(self.IsDoingMicroBodyReads)+" type:"+str(contentTypeLower)+" status:"+str(response.status_code)+" cached:"+str(isFromCache)+" for " + uri) + self.Logger.debug(self.getLogMsgPrefix() + method+" [upload:"+str(format(requestExecutionStart - self.OpenedTime, '.3f'))+"s; request_exe:"+str(format(requestExecutionEnd - requestExecutionStart, '.3f'))+"s; send:"+str(format(responseWriteDone - requestExecutionEnd, '.3f'))+"s; body_read:"+str(format(self.BodyReadTimeSec, '.3f'))+"s; compress:"+str(format(self.CompressionTimeSec, '.3f'))+"s; octo_stream_upload:"+str(format(self.ServiceUploadTimeSec, '.3f'))+"s] size:("+str(nonCompressedContentReadSizeBytes)+"->"+str(contentReadBytes)+") compressed:"+str(compressBody)+" msgcount:"+str(messageCount)+" microreads:"+str(self.IsDoingMicroBodyReads)+" type:"+str(contentTypeLower)+" status:"+str(octoHttpResult.StatusCode)+" cached:"+str(isFromCache)+" for " + uri) - def buildHeaderVector(self, builder, response): + def buildHeaderVector(self, builder, octoHttpResult:OctoHttpRequest.Result): # Gather up the headers to return. headerTableOffsets = [] - for name in response.headers: + headers = octoHttpResult.Headers + for name, value in headers.items(): nameLower = name.lower() # Since we send the entire result as one non-encoded @@ -460,7 +468,7 @@ def buildHeaderVector(self, builder, response): # Allocate strings keyOffset = builder.CreateString(name) - valueOffset = builder.CreateString(response.headers[name]) + valueOffset = builder.CreateString(value) # Create the header table HttpHeader.Start(builder) HttpHeader.AddKey(builder, keyOffset) @@ -561,7 +569,7 @@ def decompressBufferIfNeeded(self, webStreamMsg): return webStreamMsg.DataAsByteArray() - def checkForNotModifiedCacheAndUpdateResponseIfSo(self, sentHeaders, response): + def checkForNotModifiedCacheAndUpdateResponseIfSo(self, sentHeaders, octoHttpResult:OctoHttpRequest.Result): # Check if the sent headers have any conditional http headers. etag = None modifiedDate = None @@ -577,7 +585,8 @@ def checkForNotModifiedCacheAndUpdateResponseIfSo(self, sentHeaders, response): return # Look through the response headers - for key in response.headers: + headers = octoHttpResult.Headers + for key in headers: keyLower = key.lower() if etag is not None and keyLower == "etag": # Both have etags, check them. @@ -585,33 +594,33 @@ def checkForNotModifiedCacheAndUpdateResponseIfSo(self, sentHeaders, response): if etag.startswith("W/"): etag = etag[2:] # Check for an exact match. - if etag == response.headers[key]: - self.updateResponseFor304(response) + if etag == headers[key]: + self.updateResponseFor304(octoHttpResult) return if modifiedDate is not None and keyLower == "last-modified": # There are actual ways to parse and compare these, # But for now we will just do exact matches. - if modifiedDate == response.headers[key]: - self.updateResponseFor304(response) + if modifiedDate == headers[key]: + self.updateResponseFor304(octoHttpResult) return - def updateResponseFor304(self, response): + def updateResponseFor304(self, octoHttpResult:OctoHttpRequest.Result): # First of all, update the status code. - response.status_code = 304 + octoHttpResult.StatusCode = 304 # Remove any headers we don't want to send. Including some of these seems to trip up some browsers. # However, there are some we must send... # Quote - Note that the server generating a 304 response MUST generate any of the following header fields that would have been sent in a 200 (OK) response to the same request: Cache-Control, Content-Location, Date, ETag, Expires, and Vary. # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-None-Match removeHeaders = [] - for key in response.headers: + for key in octoHttpResult.Headers: keyLower = key.lower() if keyLower == "content-length": removeHeaders.append(key) if keyLower == "content-type": removeHeaders.append(key) for key in removeHeaders: - del response.headers[key] + del octoHttpResult.Headers[key] def getLogMsgPrefix(self): @@ -654,7 +663,7 @@ def shouldCompressBody(self, contentTypeLower, octoHttpResult, contentLengthOpt) # Reads data from the response body, puts it in a data vector, and returns the offset. # If the body has been fully read, this should return ogLen == 0, len = 0, and offset == None # The read style depends on the presence of the boundary string existing. - def readContentFromBodyAndMakeDataVector(self, builder, octoHttpResult, response, boundaryStr_opt, shouldCompress, contentTypeLower_NoneIfNotKnown, contentLength_NoneIfNotKnown, responseHandlerContext): + def readContentFromBodyAndMakeDataVector(self, builder, octoHttpResult:OctoHttpRequest.Result, boundaryStr_opt, shouldCompress, contentTypeLower_NoneIfNotKnown, contentLength_NoneIfNotKnown, responseHandlerContext): # This is the max size each body read will be. Since we are making local calls, most of the time # we will always get this full amount as long as theres more body to read. # Note that this amount is larger than a single read of the websocket on the server. After some testing @@ -670,14 +679,19 @@ def readContentFromBodyAndMakeDataVector(self, builder, octoHttpResult, response finalDataBuffer = None bodyReadStartSec = time.time() if self.IsUsingFullBodyBuffer: + # In this case, the entire buffer and size are known, so we get them all in one go. finalDataBuffer = octoHttpResult.FullBodyBuffer + elif self.IsUsingCustomBodyStreamCallbacks: + # In this case we just call this callback, and send whatever it sends. Note that even if this is a boundary stream, we just send back what it sends. + # If None is returned, we are done. + finalDataBuffer = octoHttpResult.GetCustomBodyStreamCallback() else: # If the boundary string exist and is not empty, we will use it to try to read the data. # Unless the self.ChunkedBodyHasNoContentLengthHeaders flag has been set, which indicate we have read the body has chunks # and failed to find any content length headers. In that case, we will just read fixed sized chunks. if self.ChunkedBodyHasNoContentLengthHeaders is False and boundaryStr_opt is not None and len(boundaryStr_opt) != 0: # Try to read a single boundary chunk - readLength = self.readStreamChunk(response, boundaryStr_opt) + readLength = self.readStreamChunk(octoHttpResult, boundaryStr_opt) # If we get a length, set the final buffer using the temp buffer. # This isn't a copy, just a reference to a subset of the buffer. if readLength != 0: @@ -689,7 +703,7 @@ def readContentFromBodyAndMakeDataVector(self, builder, octoHttpResult, response # for a number of streamed messages to fill it up. This special function does micro reads on the socket until a time limit is hit, and then # returns what was received. self.IsDoingMicroBodyReads = True - finalDataBuffer = self.doUnknownBodySizeRead(response) + finalDataBuffer = self.doUnknownBodySizeRead(octoHttpResult) else: # If there is no boundary string, but we know the content length, it's safe to just read. # This will block until either the full defaultBodyReadSizeBytes is read or the full request has been received. @@ -704,7 +718,7 @@ def readContentFromBodyAndMakeDataVector(self, builder, octoHttpResult, response else: # Use a 2mb buffer. defaultBodyReadSizeBytes = 1024 * 1024 * 1024 * 2 - finalDataBuffer = self.doBodyRead(response, defaultBodyReadSizeBytes) + finalDataBuffer = self.doBodyRead(octoHttpResult, defaultBodyReadSizeBytes) # Keep track of read times. thisBodyReadTimeSec = time.time() - bodyReadStartSec @@ -803,7 +817,7 @@ def readContentFromBodyAndMakeDataVector(self, builder, octoHttpResult, response # Reads a single chunk from the http response. # This function uses the BodyReadTempBuffer to store the data. # Returns the read size, 0 if the body read is complete. - def readStreamChunk(self, response, boundaryStr): + def readStreamChunk(self, octoHttpResult:OctoHttpRequest.Result, boundaryStr): frameSize = 0 headerSize = 0 foundContentLength = False @@ -828,7 +842,7 @@ def readStreamChunk(self, response, boundaryStr): # Read a small chunk to try to read the header # We want to read enough that hopefully we get all of the headers, but not so much that # we accidentally read two boundary messages at once. - headerBuffer = self.doBodyRead(response, 300) + headerBuffer = self.doBodyRead(octoHttpResult, 300) # If this returns 0, the body read is complete if headerBuffer is None: @@ -912,7 +926,7 @@ def readStreamChunk(self, response, boundaryStr): # Read the remainder of the chunk. if toRead > 0: - data = self.doBodyRead(response, toRead) + data = self.doBodyRead(octoHttpResult, toRead) # If we hit the end of the body, return how much we read already. if data is None: @@ -952,8 +966,14 @@ def readStreamChunk(self, response, boundaryStr): # Finally, return how much we put into the temp buffer! return tempBufferFilledSize - def doBodyRead(self, response, readSize): + + def doBodyRead(self, octoHttpResult:OctoHttpRequest.Result, readSize): try: + # Ensure there's an actual requests lib Response object to read from + response = octoHttpResult.ResponseForBodyRead + if response is None: + raise Exception("doBodyRead was called with a result that has not Response object to read from.") + # In the past we used the iter_content and such streaming function calls, but the calls had a few issues. # 1) They use a generator system where they yield data buffers. The generator has to remain referenced or the connection closes. # 2) Since the generator could only be created once, the chunk size was set on the first creation and couldn't be used. @@ -1018,7 +1038,7 @@ def doBodyRead(self, response, readSize): # there's no way to work around this. # # Ideally if we could just peak at the pending data length without blocking, we could do this much more efficiently. - def doUnknownBodySizeRead(self, response): + def doUnknownBodySizeRead(self, octoHttpResult:OctoHttpRequest.Result): # How much we will micro read, this needs to be quite small, to prevent getting "stuck" between messages. microReadSizeBytes = 300 @@ -1036,7 +1056,7 @@ def doUnknownBodySizeRead(self, response): while True: # Do a small read, which will block until the full (small) size is read. # If nothing shows up to be read, this will wait until the http request read timeout expires, and then will return None. - currentReadBuffer = self.doBodyRead(response, microReadSizeBytes) + currentReadBuffer = self.doBodyRead(octoHttpResult, microReadSizeBytes) # If None is returned, we are done. Return the current buffer or None. if currentReadBuffer is None: diff --git a/octoeverywhere/WebStream/octowebstreamwshelper.py b/octoeverywhere/WebStream/octowebstreamwshelper.py index 64334c7..26cfa83 100644 --- a/octoeverywhere/WebStream/octowebstreamwshelper.py +++ b/octoeverywhere/WebStream/octowebstreamwshelper.py @@ -49,6 +49,10 @@ def __init__(self, streamId, logger, webStream, webStreamOpenMsg, openedTime): self.IsWsObjOpened = False self.IsWsObjClosed = False + # Ensure that the http relay is enabled + if OctoHttpRequest.GetDisableHttpRelay(): + raise Exception("Web stream ws was attempted to be started when the http relay is disabled.") + # Capture the initial http context self.HttpInitialContext = webStreamOpenMsg.HttpInitialContext() if self.HttpInitialContext is None: diff --git a/octoeverywhere/commandhandler.py b/octoeverywhere/commandhandler.py index b9dcf3a..8a9a767 100644 --- a/octoeverywhere/commandhandler.py +++ b/octoeverywhere/commandhandler.py @@ -331,8 +331,7 @@ def HandleCommand(self, httpInitialContext, postBody_CanBeNone): }).encode(encoding="utf-8") # Build the full result - mockResponse = MockResponse(resultBytes, 200) - return OctoHttpRequest.Result(mockResponse, OctoStreamMsgBuilder.BytesToString(httpInitialContext.Path()), False, resultBytes) + return OctoHttpRequest.Result(200, {}, OctoStreamMsgBuilder.BytesToString(httpInitialContext.Path()), False, fullBodyBuffer=resultBytes) # The goal here is to keep as much of the common logic as common as possible. @@ -354,7 +353,6 @@ def ProcessCommand(self, commandPath, jsonObj_CanBeNone): elif commandPathLower.startswith("cancel"): return self.Cancel() - return CommandResponse.Error(CommandHandler.c_CommandError_UnknownCommand, "The command path didn't match any known commands.") @@ -377,28 +375,3 @@ def __init__(self, statusCode, resultDict, errorStr_CanBeNull): self.StatusCode = statusCode self.ResultDict = resultDict self.ErrorStr = errorStr_CanBeNull - - -class MockResponse(): - - def __init__(self, fullBodyBytes, statusCode): - # - # The following are public members that must exist to mock the request lib's response object. - # - - # All values in this dict must be strings. - self.headers = { - "Content-Length": str(len(fullBodyBytes)), # Use this so the full size is known and read at once. - "Content-Type": "application/json", # Use this so we get compressed - } - self.status_code = statusCode - - - # Needed to mock the with keyword - def __enter__(self): - return self - - - # Needed to mock the with keyword - def __exit__(self, t, v, tb): - pass diff --git a/octoeverywhere/notificationshandler.py b/octoeverywhere/notificationshandler.py index c547921..c603e9b 100644 --- a/octoeverywhere/notificationshandler.py +++ b/octoeverywhere/notificationshandler.py @@ -181,37 +181,37 @@ def IsTrackingPrint(self) -> bool: return self._IsPingTimerRunning() - # A special case used by moonraker to restore the state of an ongoing print that we don't know of. - # What we want to do is check moonraker's current state and our current state, to see if there's anything that needs to be synced. + # A special case used by moonraker and bambu to restore the state of an ongoing print that we don't know of. + # What we want to do is check moonraker or bambu's current state and our current state, to see if there's anything that needs to be synced. # Remember that we might be syncing because our service restarted during a print, or moonraker restarted, so we might already have # the correct context. # # Most importantly, we want to make sure the ping timer and thus Gadget get restored to the correct states. # - def OnRestorePrintIfNeeded(self, moonrakerPrintStatsState, fileName_CanBeNone, totalDurationFloatSec_CanBeNone): - if moonrakerPrintStatsState == "printing": + def OnRestorePrintIfNeeded(self, isPrinting:bool, isPaused:bool, fileName_CanBeNone, totalDurationFloatSec_CanBeNone): + if isPrinting: # There is an active print. Check our state. if self._IsPingTimerRunning(): - self.Logger.info("Moonraker client sync state: Detected an active print and our timers are already running, there's nothing to do.") + self.Logger.info("Restore client sync state: Detected an active print and our timers are already running, there's nothing to do.") return else: - self.Logger.info("Moonraker client sync state: Detected an active print but we aren't tracking it, so we will restore now.") + self.Logger.info("Restore client sync state: Detected an active print but we aren't tracking it, so we will restore now.") # We need to do the restore of a active print. - elif moonrakerPrintStatsState == "paused": + elif isPaused: # There is a print currently paused, check to see if we have a filename, which indicates if we know of a print or not. if self._HasCurrentPrintFileName(): - self.Logger.info("Moonraker client sync state: Detected a paused print, but we are already tracking a print, so there's nothing to do.") + self.Logger.info("Restore client sync state: Detected a paused print, but we are already tracking a print, so there's nothing to do.") return else: - self.Logger.info("Moonraker client sync state: Detected a paused print, but we aren't tracking any prints, so we will restore now") + self.Logger.info("Restore client sync state: Detected a paused print, but we aren't tracking any prints, so we will restore now") else: # There's no print running. if self._IsPingTimerRunning(): - self.Logger.info("Moonraker client sync state: Detected no active print but our ping timers ARE RUNNING. Stopping them now.") + self.Logger.info("Restore client sync state: Detected no active print but our ping timers ARE RUNNING. Stopping them now.") self.StopTimers() return else: - self.Logger.info("Moonraker client sync state: Detected no active print and no ping timers are running, so there's nothing to do.") + self.Logger.info("Restore client sync state: Detected no active print and no ping timers are running, so there's nothing to do.") return # If we are here, we need to restore a print. @@ -235,7 +235,7 @@ def OnRestorePrintIfNeeded(self, moonrakerPrintStatsState, fileName_CanBeNone, t self.RestorePrintProgressPercentage = True # Make sure the timers are set correctly - if moonrakerPrintStatsState == "printing": + if isPrinting: # If we have a total duration, use it to offset the "hours reported" so our time based notifications # are correct. hoursReportedInt = 0 @@ -244,7 +244,7 @@ def OnRestorePrintIfNeeded(self, moonrakerPrintStatsState, fileName_CanBeNone, t hoursReportedInt = int(math.floor(totalDurationFloatSec_CanBeNone / 60.0 / 60.0)) # Setup the timers, with hours reported, to make sure that the ping timer and Gadget are running. - self.Logger.info("Moonraker client sync state: Restoring printing timer with existing duration of "+str(totalDurationFloatSec_CanBeNone)) + self.Logger.info("Restore client sync state: Restoring printing timer with existing duration of "+str(totalDurationFloatSec_CanBeNone)) self.StartPrintTimers(False, hoursReportedInt) else: # On paused, make sure they are stopped. @@ -704,7 +704,7 @@ def GetNotificationSnapshot(self, snapshotResizeParams = None): octoHttpResponse = WebcamHelper.Get().GetSnapshot() # Check for a valid response. - if octoHttpResponse is None or octoHttpResponse.Result is None or octoHttpResponse.Result.status_code != 200: + if octoHttpResponse is None or octoHttpResponse.StatusCode != 200: return None # GetSnapshot will always return the full result already read. diff --git a/octoeverywhere/octohttprequest.py b/octoeverywhere/octohttprequest.py index 1e263c6..4713617 100644 --- a/octoeverywhere/octohttprequest.py +++ b/octoeverywhere/octohttprequest.py @@ -12,6 +12,7 @@ class OctoHttpRequest: LocalHttpProxyIsHttps = False LocalOctoPrintPort = 5000 LocalHostAddress = "127.0.0.1" + DisableHttpRelay = False @staticmethod def SetLocalHttpProxyPort(port): @@ -41,6 +42,13 @@ def SetLocalHostAddress(address): def GetLocalhostAddress(): return OctoHttpRequest.LocalHostAddress + @staticmethod + def SetDisableHttpRelay(disableHttpRelay:bool): + OctoHttpRequest.DisableHttpRelay = disableHttpRelay + @staticmethod + def GetDisableHttpRelay() -> bool: + return OctoHttpRequest.DisableHttpRelay + # Based on the URL passed, this will return PathTypes.Relative or PathTypes.Absolute @staticmethod @@ -51,58 +59,125 @@ def GetPathType(url): # TODO - It might be worth to add some logic to try to detect no protocol hostnames, like test.com/helloworld. return PathTypes.Relative - # The result of a successfully made http request. - # "successfully made" means we talked to the server, not the the http - # response is good. + + # This result class is a wrapper around the requests PY lib Response object. + # For the most part, it should abstract away what's needed from the Response object, so that an actual Response object isn't needed + # for all http calls. However, sometimes the actual Response object might be in this result object, because a ref to it needs to be held + # so the body stream can be read, assuming there's no full body buffer. # - # FullBodyBuffer defaults to None. But if it's set, it should be used instead of reading from the http response body. + # There are three ways this class can contain a body to be used. + # 1) ResponseForBodyRead - If this is not None, then there's a requests.Response attached to this Result and it can be used to be read from. + # Note, in this case, ideally the Result is used with a `with` keyword to cleanup when it's done. + # 2) FullBodyBuffer - If this is not None, then there's a fully read body buffer that should be used. + # In this case, the size of the body is known, it's the size of the full body buffer. The size can't change. + # 3) CustomBodyStream - If this is not None, then there's a custom body stream that should be used. + # This callback can be implemented by anything. The size is unknown and should continue until the callback returns None. + # customBodyStreamCallback() -> byteArray : Called to get more bytes. If None is returned, the stream is done. + # customBodyStreamClosedCallback() -> None : MUST BE CALLED when this Result object is closed, to clean up the stream. class Result(): - def __init__(self, result, url, didFallback, fullBodyBuffer=None): - self.result = result - self.url:str = url - self.didFallback:bool = didFallback - self.fullBodyBuffer = fullBodyBuffer - self.isZlibCompressed:bool = False - self.fullBodyBufferPreCompressedSize:int = 0 + def __init__(self, statusCode:int, headers:dict, url:str, didFallback:bool, fullBodyBuffer=None, requestLibResponseObj:requests.Response=None, customBodyStreamCallback=None, customBodyStreamClosedCallback=None): + self._statusCode = statusCode + self._headers = headers + self._url:str = url + self._requestLibResponseObj = requestLibResponseObj + self._didFallback:bool = didFallback + self._fullBodyBuffer = fullBodyBuffer + self._isZlibCompressed:bool = False + self._fullBodyBufferPreCompressedSize:int = 0 + self.SetFullBodyBuffer(fullBodyBuffer) + self._customBodyStreamCallback = customBodyStreamCallback + self._customBodyStreamClosedCallback = customBodyStreamClosedCallback + if (self._customBodyStreamCallback is not None and self._customBodyStreamClosedCallback is None) or (self._customBodyStreamCallback is None and self._customBodyStreamClosedCallback is not None): + raise Exception("Both the customBodyStreamCallback and customBodyStreamClosedCallback must be set!") @property - def Result(self): - return self.result + def StatusCode(self) -> int: + return self._statusCode @property - def Url(self): - return self.url + def Headers(self) -> dict: + return self._headers @property - def DidFallback(self): - return self.didFallback + def Url(self) -> str: + return self._url @property - def FullBodyBuffer(self): + def DidFallback(self) -> bool: + return self._didFallback + + # This should only be used for reading the http stream body and it might be None + # If this Result was created without one. + @property + def ResponseForBodyRead(self) -> requests.Response: + return self._requestLibResponseObj + + @property + def FullBodyBuffer(self) -> bytearray: # Defaults to None - return self.fullBodyBuffer + return self._fullBodyBuffer @property - def IsBodyBufferZlibCompressed(self): + def IsBodyBufferZlibCompressed(self) -> bool: # There must be a buffer and the flag must be set. - return self.isZlibCompressed and self.fullBodyBuffer is not None + return self._isZlibCompressed and self._fullBodyBuffer is not None @property - def BodyBufferPreCompressSize(self): + def BodyBufferPreCompressSize(self) -> int: # There must be a buffer - if self.fullBodyBuffer is None: + if self._fullBodyBuffer is None: return 0 - return self.fullBodyBufferPreCompressedSize + return self._fullBodyBufferPreCompressedSize # Note the buffer can be bytes or bytearray object! # A bytes object is more efficient, but bytearray can be edited. def SetFullBodyBuffer(self, buffer, isZlibCompressed:bool = False, preCompressedSize:int = 0): - self.fullBodyBuffer = buffer - self.isZlibCompressed = isZlibCompressed - self.fullBodyBufferPreCompressedSize = preCompressedSize + self._fullBodyBuffer = buffer + self._isZlibCompressed = isZlibCompressed + self._fullBodyBufferPreCompressedSize = preCompressedSize if isZlibCompressed and preCompressedSize <= 0: raise Exception("The pre-compression full size must be set if the buffer is compressed.") + # Since most things use request Stream=True, this is a helpful util that will read the entire + # content of a request and return it. Note if the request has no defined length, this will read + # as long as the stream will go. + def ReadAllContentFromStreamResponse(self) -> None: + # Ensure we have a stream to read. + if self._requestLibResponseObj is None: + raise Exception("ReadAllContentFromStreamResponse was called on a result with no request lib Response object.") + buffer = None + # We can't simply use response.content, since streaming was enabled. + # We need to use iter_content, since it will keep returning data until all is read. + # We use a high chunk count, so most of the time it will read all of the content in one go. + for chunk in self._requestLibResponseObj.iter_content(10000000): + if buffer is None: + buffer = chunk + else: + buffer += chunk + self.SetFullBodyBuffer(buffer) + + @property + def GetCustomBodyStreamCallback(self): + return self._customBodyStreamCallback + + @property + def GetCustomBodyStreamClosedCallback(self): + return self._customBodyStreamClosedCallback + + # We need to support the with keyword incase we have an actual Response object. + def __enter__(self): + if self._requestLibResponseObj is not None: + self._requestLibResponseObj.__enter__() + return self + + # We need to support the with keyword incase we have an actual Response object. + def __exit__(self, t, v, tb): + if self._requestLibResponseObj is not None: + self._requestLibResponseObj.__exit__(t, v, tb) + if self._customBodyStreamClosedCallback is not None: + self._customBodyStreamClosedCallback() + + # Handles making all http calls out of the plugin to OctoPrint or other services running locally on the device or # even on other devices on the LAN. # @@ -124,7 +199,7 @@ def MakeHttpCallOctoStreamHelper(logger, httpInitialContext, method, headers, da # The X-Forwarded-Host header will tell the OctoPrint server the correct place to set the location redirect header. # However, for calls that aren't proxy calls, things like local snapshot requests and such, we want to allow redirects to be more robust. @staticmethod - def MakeHttpCall(logger, pathOrUrl, pathOrUrlType, method, headers, data=None, allowRedirects=False): + def MakeHttpCall(logger, pathOrUrl, pathOrUrlType, method, headers, data=None, allowRedirects=False) -> Result: # First of all, we need to figure out what the URL is. There are two options # # 1) Absolute URLs @@ -350,14 +425,14 @@ def MakeHttpCallAttempt(logger, attemptName, method, url, headers, data, mainRes if response is not None and response.status_code != 404: # We got a valid response, we are done. # Return true and the result object, so it can be returned. - return OctoHttpRequest.AttemptResult(True, OctoHttpRequest.Result(response, url, isFallback)) + return OctoHttpRequest.AttemptResult(True, OctoHttpRequest.Result(response.status_code, response.headers, url, isFallback, requestLibResponseObj=response)) # Check if we have another fallback URL to try. if nextFallbackUrl is not None: # We have more fallbacks to try. # Return false so we keep going, but also return this response if we had one. This lets # use capture the main result object, so we can use it eventually if all fallbacks fail. - return OctoHttpRequest.AttemptResult(False, OctoHttpRequest.Result(response, url, isFallback)) + return OctoHttpRequest.AttemptResult(False, OctoHttpRequest.Result(response.status_code, response.headers, url, isFallback, requestLibResponseObj=response)) # We don't have another fallback, so we need to end this. if mainResult is not None: diff --git a/octoeverywhere/requestsutils.py b/octoeverywhere/requestsutils.py deleted file mode 100644 index bb8df6b..0000000 --- a/octoeverywhere/requestsutils.py +++ /dev/null @@ -1,18 +0,0 @@ - -class RequestsUtils: - - # Since most things use request Stream=True, this is a helpful util that will read the entire - # content of a request and return it. Note if the request has no defined length, this will read - # as long as the stream will go. - @staticmethod - def ReadAllContentFromStreamResponse(response): - buffer = None - # We can't simply use response.content, since streaming was enabled. - # We need to use iter_content, since it will keep returning data until all is read. - # We use a high chunk count, so most of the time it will read all of the content in one go. - for chunk in response.iter_content(10000000): - if buffer is None: - buffer = chunk - else: - buffer += chunk - return buffer diff --git a/octoeverywhere/webcamhelper.py b/octoeverywhere/webcamhelper.py index b56b8fc..a90d566 100644 --- a/octoeverywhere/webcamhelper.py +++ b/octoeverywhere/webcamhelper.py @@ -4,7 +4,6 @@ from .sentry import Sentry from .octohttprequest import OctoHttpRequest -from .requestsutils import RequestsUtils # # A platform agnostic definition of a webcam stream. @@ -164,6 +163,10 @@ def GetWebcamStream(self, cameraName:str = None) -> OctoHttpRequest.Result: def _GetWebcamStreamInternal(self, cameraName:str) -> OctoHttpRequest.Result: + # Check if the platform helper has an override. If so, it is responsible for all of the stream getting logic. + if hasattr(self.WebcamPlatformHelperInterface, 'GetStream_Override'): + return self.WebcamPlatformHelperInterface.GetStream_Override(cameraName) + # Try to get the URL from the settings. webcamStreamUrl = self.GetWebcamStreamUrl(cameraName) if webcamStreamUrl is not None: @@ -191,6 +194,10 @@ def GetSnapshot(self, cameraName:str = None) -> OctoHttpRequest.Result: def _GetSnapshotInternal(self, cameraName:str) -> OctoHttpRequest.Result: + # Check if the platform helper has an override. If so, it is responsible for all of the snapshot getting logic. + if hasattr(self.WebcamPlatformHelperInterface, 'GetSnapshot_Override'): + return self.WebcamPlatformHelperInterface.GetSnapshot_Override(cameraName) + # First, try to get the snapshot using the string defined in settings. snapshotUrl = self.GetSnapshotUrl(cameraName) if snapshotUrl is not None: @@ -201,7 +208,8 @@ def _GetSnapshotInternal(self, cameraName:str) -> OctoHttpRequest.Result: self.Logger.debug("Trying to get a snapshot using url: %s", snapshotUrl) octoHttpResult = OctoHttpRequest.MakeHttpCall(self.Logger, snapshotUrl, OctoHttpRequest.GetPathType(snapshotUrl), "GET", {}, allowRedirects=True) # If the result was successful, we are done. - if octoHttpResult is not None and octoHttpResult.Result is not None and octoHttpResult.Result.status_code == 200: + + if octoHttpResult is not None and octoHttpResult.StatusCode == 200: return octoHttpResult # If getting the snapshot from the snapshot URL fails, try getting a single frame from the mjpeg stream @@ -219,28 +227,25 @@ def _GetSnapshotFromStream(self, url) -> OctoHttpRequest.Result: # We use the allow redirects flag to make the API more robust, since some webcam images might need that. self.Logger.debug("_GetSnapshotFromStream - Trying to get a snapshot using THE STREAM URL: %s", url) octoHttpResult = OctoHttpRequest.MakeHttpCall(self.Logger, url, OctoHttpRequest.GetPathType(url), "GET", {}, allowRedirects=True) - if octoHttpResult is None or octoHttpResult.Result is None: + if octoHttpResult is None: self.Logger.debug("_GetSnapshotFromStream - Failed to make web request.") return None # Check for success. - response = octoHttpResult.Result - if response is None or response.status_code != 200: - if response is None: - self.Logger.info("Snapshot fallback failed due to the http call having no response object.") - else: - self.Logger.info("Snapshot fallback failed due to the http call having a bad status: "+str(response.status_code)) + if octoHttpResult.StatusCode != 200: + self.Logger.info("Snapshot fallback failed due to the http call having a bad status: "+str(octoHttpResult.StatusCode)) return None # Hold the entire response in a with block, so that we we leave it will be cleaned up, since it's most likely a streaming stream. - with response: + with octoHttpResult: # We expect this to be a multipart stream if it's going to be a mjpeg stream. isMultipartStream = False contentTypeLower = "" - for name in response.headers: + headers = octoHttpResult.Headers + for name in headers: nameLower = name.lower() if nameLower == "content-type": - contentTypeLower = response.headers[name].lower() + contentTypeLower = headers[name].lower() if contentTypeLower.startswith("multipart/"): isMultipartStream = True break @@ -250,9 +255,15 @@ def _GetSnapshotFromStream(self, url) -> OctoHttpRequest.Result: self.Logger.info("Snapshot fallback failed not correct content type: "+str(contentTypeLower)) return None + # Ensure we have a response object to read from. + responseForBodyRead = octoHttpResult.ResponseForBodyRead + if responseForBodyRead is None: + self.Logger.warn("Snapshot fallback got a response that didn't have a requests lib Response object to read from.") + return None + # Try to read some of the stream, so we can find the content type and the size of this first frame. # We use the raw response, so we can control directly how much we read. - dataBuffer = response.raw.read(300) + dataBuffer = responseForBodyRead.raw.read(300) if dataBuffer is None: self.Logger.info("Snapshot fallback failed no data returned.") return None @@ -304,7 +315,7 @@ def _GetSnapshotFromStream(self, url) -> OctoHttpRequest.Result: totalDesiredBufferSize = frameSizeInt + headerStrSize toRead = totalDesiredBufferSize - len(dataBuffer) if toRead > 0: - data = response.raw.read(toRead) + data = responseForBodyRead.raw.read(toRead) if data is None: self.Logger.error("_GetSnapshotFromStream failed to read the rest of the image buffer.") return None @@ -313,7 +324,7 @@ def _GetSnapshotFromStream(self, url) -> OctoHttpRequest.Result: # Since this is a stream, ideally we close it as soon as possible to not waste resources. # Otherwise this will be auto closed when the function leaves, since we are using the with: scope try: - response.close() + responseForBodyRead.close() except Exception: pass @@ -332,17 +343,16 @@ def _GetSnapshotFromStream(self, url) -> OctoHttpRequest.Result: if len(imageBuffer) != frameSizeInt: self.Logger.warn("Snapshot callback final image size was not the frame size. expected: "+str(frameSizeInt)+", got: "+str(len(imageBuffer))) - # If successful, we will use the already existing response object but update the values to match the fixed size body and content type. - response.status_code = 200 - # Clear all of the current - response.headers.clear() - # Set the content type to the header we got from the stream chunk. - response.headers["content-type"] = contentType - # It's very important this size matches the body buffer we give OctoHttpRequest, or the logic in the http loop will fail because it will keep trying to read more. - response.headers["content-length"] = str(len(imageBuffer)) + # If successful, set values to match the fixed size body and content type. + headers = { + # Set the content type to the header we got from the stream chunk. + "content-type": contentType, + # It's very important this size matches the body buffer we give OctoHttpRequest, or the logic in the http loop will fail because it will keep trying to read more. + "content-length": str(len(imageBuffer)) + } # Return a result. Return the full image buffer which will be used as the response body. self.Logger.debug("Successfully got image from stream URL. Size: %s, Format: %s", str(len(imageBuffer)), contentType) - return OctoHttpRequest.Result(response, url, True, imageBuffer) + return OctoHttpRequest.Result(200, headers, url, True, fullBodyBuffer=imageBuffer) except ConnectionError as e: # We have a lot of telemetry indicating a read timeout can happen while trying to read from the stream # in that case we should just get out of here. @@ -397,7 +407,7 @@ def ListWebcams(self): # Checks if the result was success and if so adds the common header. # Returns the octoHttpResult, so the function is chainable def _AddOeWebcamTransformHeader(self, octoHttpResult, cameraName:str): - if octoHttpResult is None or octoHttpResult.Result is None: + if octoHttpResult is None: return octoHttpResult # Default to none @@ -415,7 +425,7 @@ def _AddOeWebcamTransformHeader(self, octoHttpResult, cameraName:str): transformStr += "rotate="+str(settings.Rotation)+" " # Set the header - octoHttpResult.Result.headers[WebcamHelper.c_OeWebcamTransformHeaderKey] = transformStr + octoHttpResult.Headers[WebcamHelper.c_OeWebcamTransformHeaderKey] = transformStr return octoHttpResult @@ -425,18 +435,21 @@ def _AddOeWebcamTransformHeader(self, octoHttpResult, cameraName:str): # To combat this, we will check if the image is a jpeg, and if so, ensure the header is set correctly. # # Returns the octoHttpResult, so the function is chainable - def _EnsureJpegHeaderInfo(self, octoHttpResult: OctoHttpRequest.Result): + def _EnsureJpegHeaderInfo(self, octoHttpResult:OctoHttpRequest.Result): # Ensure we got a result. - if octoHttpResult is None or octoHttpResult.Result is None: + if octoHttpResult is None: return octoHttpResult # The GetSnapshot API will always return the fully buffered snapshot. - buf = RequestsUtils.ReadAllContentFromStreamResponse(octoHttpResult.Result) + # If there already isn't a full buffered body, make one now. + buf = octoHttpResult.FullBodyBuffer if buf is None: - self.Logger.error("_EnsureJpegHeaderInfo got a null body read from ReadAllContentFromStreamResponse") - return None - # Set the buffer now, incase of early returns. - octoHttpResult.SetFullBodyBuffer(buf) + # This will read the entire stream and store it into the FullBodyBuffer + octoHttpResult.ReadAllContentFromStreamResponse() + buf = octoHttpResult.FullBodyBuffer + if buf is None: + self.Logger.error("_EnsureJpegHeaderInfo got a null body read from ReadAllContentFromStreamResponse") + return None # Handle the buffer. # In a nutshell, all jpeg images have a lot of header segments, but they must have the app0 header. diff --git a/octoprint_octoeverywhere/slipstream.py b/octoprint_octoeverywhere/slipstream.py index 99b7b59..4fb05bc 100644 --- a/octoprint_octoeverywhere/slipstream.py +++ b/octoprint_octoeverywhere/slipstream.py @@ -1,7 +1,6 @@ import threading import time import zlib -import sys from octoeverywhere.sentry import Sentry from octoeverywhere.compat import Compat @@ -259,17 +258,18 @@ def _GetCacheReadyOctoHttpResult(self, url): octoHttpResult = OctoHttpRequest.MakeHttpCall(self.Logger, url, PathTypes.Relative, "GET", headers) # Check for success - if octoHttpResult is None or octoHttpResult.Result is None or octoHttpResult.Result.status_code != 200: + if octoHttpResult is None or octoHttpResult.StatusCode != 200: self.Logger.error("Slipstream failed to make the http request for "+url) return None # Remove any headers we don't want. setCookieKey = None contentLength = None - for key in octoHttpResult.Result.headers: + headers = octoHttpResult.Headers + for key in headers: keyLower = key.lower() if keyLower == "content-length": - contentLength = int(octoHttpResult.Result.headers[key]) + contentLength = int(headers[key]) elif keyLower == "set-cookie": setCookieKey = key @@ -280,18 +280,18 @@ def _GetCacheReadyOctoHttpResult(self, url): # We remove Set-Cookie so no session gets applied from this cached item. if setCookieKey is not None: - del octoHttpResult.Result.headers[setCookieKey] + del octoHttpResult.Headers[setCookieKey] # Set the cache header - octoHttpResult.Result.headers["x-oe-slipstream-plugin"] = "1" + octoHttpResult.Headers["x-oe-slipstream-plugin"] = "1" # Since the request is setup to use streaming mode, we need to read in the entire contents and store them # because when the function exist the request will be closed and the stream will no longer be able to be read. # This is actually a good idea, so the request connection doesn't hang around for a long time. - buffer = bytearray() + buffer = None try: - for data in octoHttpResult.Result.iter_content(chunk_size=contentLength): - buffer += data + octoHttpResult.ReadAllContentFromStreamResponse() + buffer = octoHttpResult.FullBodyBuffer except Exception as e: self.Logger.error("Slipstream failed to read index buffer for "+url+", e:"+str(e)) return None @@ -301,17 +301,6 @@ def _GetCacheReadyOctoHttpResult(self, url): self.Logger.error("Slipstream read a a body of different size then the content length. url:"+url+" body:"+str(len(buffer))+" cl:"+str(contentLength)) return None - # Since we are doing this in the background, we will take some time to compress it. - # This has two benefits: - # 1) We don't need to spend time and CPU cycles compressing on the fly. - # 2) We can do a better compression to get less size that we don't have time for in realtime. - # Note for now, we compress everything. - # - # PY2 zlib.compress can't accept a bytearray, so we must convert them before compressing. - # This isn't ideal, but not a big deal since this is in the background. - if sys.version_info[0] < 3: - buffer = bytes(buffer) - # Do the compression. # See the compression chat in the main http stream class for tradeoffs about complexity. ogSize = len(buffer) diff --git a/py_installer/Logging.py b/py_installer/Logging.py index 5782c59..7791704 100644 --- a/py_installer/Logging.py +++ b/py_installer/Logging.py @@ -1,4 +1,5 @@ import os +import logging from datetime import datetime # pylint: disable=import-error # Only exists on linux import pwd @@ -19,7 +20,7 @@ class Logger: IsDebugEnabled = False OutputFile = None OutputFilePath = None - + PyLogger = None @staticmethod def InitFile(userHomePath:str, userName:str): @@ -48,6 +49,16 @@ def Finalize(): pass + # Returns a logging.Logger standard logger which can be used in the common PY files. + @staticmethod + def GetPyLogger() -> logging.Logger: + if Logger.PyLogger is None: + Logger.PyLogger = logging.getLogger("octoeverywhere-installer") + Logger.PyLogger.setLevel(logging.DEBUG) + Logger.PyLogger.addHandler(CustomLogHandler()) + return Logger.PyLogger + + @staticmethod def DeleteLogFile(): try: @@ -110,3 +121,18 @@ def _WriteToFile(level:str, msg:str): Logger.OutputFile.write(str(datetime.now()) + " ["+level+"] - " + msg+"\n") except Exception: pass + + +# Allows us to return a logging.Logger for use in common classes. +class CustomLogHandler(logging.Handler): + def emit(self, record:logging.LogRecord): + if record.levelno == logging.DEBUG: + Logger.Debug(record.getMessage()) + elif record.levelno == logging.INFO: + Logger.Info(record.getMessage()) + elif record.levelno == logging.WARNING: + Logger.Warn(record.getMessage()) + elif record.levelno == logging.ERROR: + Logger.Error(record.getMessage()) + else: + Logger.Info("Unknown logging level "+record.getMessage()) diff --git a/py_installer/NetworkConnectors/BambuConnector.py b/py_installer/NetworkConnectors/BambuConnector.py index a81eeb1..82bd40a 100644 --- a/py_installer/NetworkConnectors/BambuConnector.py +++ b/py_installer/NetworkConnectors/BambuConnector.py @@ -1,42 +1,40 @@ -import threading -import socket - -from octoeverywhere.websocketimpl import Client +from linux_host.networksearch import NetworkSearch from py_installer.Util import Util from py_installer.Logging import Logger from py_installer.Context import Context from py_installer.ConfigHelper import ConfigHelper - # A class that helps the user discover, connect, and setup the details required to connect to a remote Bambu Labs printer. class BambuConnector: + def EnsureBambuConnection(self, context:Context): Logger.Debug("Running bambu connect ensure config logic.") # For Bambu printers, we need the IP or Hostname, the port is static, # and we also need the printer SN and access token. ip, port = ConfigHelper.TryToGetCompanionDetails(context) - accessToken, printerSn = ConfigHelper.TryToGetBambuData(context) - if ip is not None and port is not None and accessToken is not None and printerSn is not None: + accessCode, printerSn = ConfigHelper.TryToGetBambuData(context) + if ip is not None and port is not None and accessCode is not None and printerSn is not None: # Check if we can still connect. This can happen if the IP address changes, the user might need to setup the printer again. Logger.Info(f"Existing bambu config found. IP: {ip} - {printerSn}") Logger.Info("Checking if we can connect to your Bambu Labs printer...") - #success, _ = self._CheckForMoonraker(ip, port, 10.0) - success = True - if success: + result = NetworkSearch.ValidateConnection_Bambu(Logger.GetPyLogger(), ip, accessCode, printerSn, portStr=port, timeoutSec=10.0) + if result.Success(): Logger.Info("Successfully connected to you Bambu Labs printer!") return else: # Let the user keep this connection setup, or try to set it up again. Logger.Blank() - Logger.Warn(f"No connection found using the IP {ip}.") + Logger.Warn(f"We failed to connect or authenticate to your printer using {ip}.") if Util.AskYesOrNoQuestion("Do you want to setup the Bambu Labs printer connection again?") is False: Logger.Info(f"Keeping the existing Bambu Labs printer connection setup. {ip} - {printerSn}") return ipOrHostname, port, accessToken, printerSn = self._SetupNewBambuConnection() + Logger.Info(f"You Bambu printer was found and authentication was successful! IP: {ip}") + ConfigHelper.WriteCompanionDetails(context, ipOrHostname, port) ConfigHelper.WriteBambuDetails(context, accessToken, printerSn) Logger.Blank() @@ -47,213 +45,169 @@ def EnsureBambuConnection(self, context:Context): # Helps the user setup a bambu connection via auto scanning or manual setup. # Returns (ip:str, port:str, accessToken:str, printerSn:str) def _SetupNewBambuConnection(self): - Logger.Blank() - Logger.Blank() - Logger.Blank() - Logger.Header("##################################") - Logger.Header(" Bambu Labs Printer Setup") - Logger.Header("##################################") - Logger.Blank() - Logger.Info("For OctoEverywhere Bambu Connect to work, it needs to know how to connect to your Bambu Labs printer.") - Logger.Info("If you have any trouble, we are happy to help! Contact us at support@octoeverywhere.com") - Logger.Blank() - ipOrHostname = input("Enter the IP or Hostname: ") - accessToken = input("Enter the Access Token: ") - printerSn = input("Enter the printer's serial number: ") - return (ipOrHostname, "8883", accessToken, printerSn) - # Logger.Info("Searching for local Klipper printers... please wait... (about 5 seconds)") - # foundIps = self._ScanForMoonrakerInstances() - # if len(foundIps) > 0: - # # Sort them so they present better. - # foundIps = sorted(foundIps) - # Logger.Blank() - # Logger.Info("Klipper was found on the following IP addresses:") - # count = 0 - # for ip in foundIps: - # count += 1 - # Logger.Info(f" {count}) {ip}:7125") - # Logger.Blank() - # while True: - # response = input("Enter the number next to the Klipper instance you want to use or enter `m` to manually setup the connection: ") - # response = response.lower().strip() - # if response == "m": - # # Break to fall through to the manual setup. - # break - # try: - # # Parse the input and -1 it, so it aligns with the array length. - # tempInt = int(response.lower().strip()) - 1 - # if tempInt >= 0 and tempInt < len(foundIps): - # return (foundIps[tempInt], "7125") - # except Exception as _: - # Logger.Warn("Invalid input, try again.") - # else: - # Logger.Info("No local Klipper devices could be automatically found.") - - # # Do the manual setup process. - # ipOrHostname = "" - # port = "7125" - # while True: - # try: - # Logger.Blank() - # Logger.Blank() - # Logger.Info("Please enter the IP address or Hostname of the device running Klipper/Moonraker/Mainsail/Fluidd.") - # Logger.Info("The IP address might look something like `192.168.1.5` or a Hostname might look like `klipper.local`") - # ipOrHostname = input("Enter the IP or Hostname: ") - # # Clean up what the user entered. - # ipOrHostname = ipOrHostname.lower().strip() - # if ipOrHostname.find("://") != -1: - # ipOrHostname = ipOrHostname[ipOrHostname.find("://")+3:] - # if ipOrHostname.find("/") != -1: - # ipOrHostname = ipOrHostname[:ipOrHostname.find("/")] - - # Logger.Blank() - # Logger.Info("Please enter the port Moonraker is running on.") - # Logger.Info("If you don't know the port or want to use the default port (7125), press enter.") - # port = input("Enter Moonraker Port: ") - # if len(port) == 0: - # port = "7125" - - # Logger.Blank() - # Logger.Info(f"Trying to connect to Moonraker via {ipOrHostname}:{port} ...") - # success, exception = self._CheckForMoonraker(ipOrHostname, port, 10.0) - - # # Handle the result. - # if success: - # return (ipOrHostname, port) - # else: - # Logger.Blank() - # Logger.Blank() - # if exception is not None: - # Logger.Error("Klipper connection failed.") - # else: - # Logger.Error("Klipper connection timed out.") - # Logger.Warn("Make sure the device is powered on, has an network connection, and the ip is correct.") - # if exception is not None: - # Logger.Warn(f"Error {str(exception)}") - # except Exception as e: - # Logger.Warn("Failed to setup Klipper, try again. "+str(e)) - - - # Given an ip or hostname and port, this will try to detect if there's a moonraker instance. - # Returns (success:, exception | None) - def _CheckForMoonraker(self, ip:str, port:str, timeoutSec:float = 5.0): - doneEvent = threading.Event() - lock = threading.Lock() - result = {} - - # Create the URL - url = f"ws://{ip}:{port}/websocket" + while True: + Logger.Blank() + Logger.Blank() + Logger.Blank() + Logger.Header("##################################") + Logger.Header(" Bambu Labs Printer Setup") + Logger.Header("##################################") + Logger.Blank() + Logger.Info("OctoEverywhere Bambu Connect needs to connect to your Bambu Labs printer to provide remote access.") + Logger.Info("Bambu Connect needs your printer's Access Code and Serial Number to connect to your printer.") + Logger.Info("If you have any trouble, we are happy to help! Contact us at support@octoeverywhere.com") + + # Get the access code. + accessCode = None + while True: + Logger.Blank() + Logger.Blank() + Logger.Header("We need your Bambu Labs printer's Access Code to connect.") + Logger.Info("The Access Code can be found using the screen on your printer, in the Network settings.") + Logger.Blank() + Logger.Warn("Follow this link for a step-by-step guide to find the Access Code for your printer:") + Logger.Warn("https://octoeverywhere.com/s/access-code") + Logger.Blank() + accessCode = input("Enter your printer's Access Code: ") + + # Validate + accessCode = accessCode.strip() + if len(accessCode) != 8: + if Util.AskYesOrNoQuestion(f"The Access Code should be 8 numbers, you have entered {len(accessCode)}. Do you want to try again? "): + continue + + retryEntry = False + for c in accessCode: + if not c.isdigit(): + if Util.AskYesOrNoQuestion("The Access Code should only be numbers, you seem to have entered something else. Do you want to try again? "): + retryEntry = True + break + if retryEntry: + continue + + # Accept the input. + break + + + Logger.Blank() + Logger.Blank() + Logger.Blank() + + # Get the serial number. + printerSn = None + while True: + Logger.Blank() + Logger.Header("Finally, Bambu Connect needs your Bambu Labs printer's Serial Number to connect.") + Logger.Info("The Serial Number is required for authentication when the printer's local network protocol.") + Logger.Info("Your Serial Number and Access Code are only stored on this device and will not be uploaded.") + Logger.Blank() + Logger.Warn("Follow this link for a step-by-step guide to find the Serial Number for your printer:") + Logger.Warn("https://octoeverywhere.com/s/bambu-sn") + Logger.Blank() + printerSn = input("Enter your printer's Serial Number: ") + + # The SN should always be upper case letters. + printerSn = printerSn.strip().upper() + + # Validate + # It seems the SN are 15 digits + if len(printerSn) != 15: + if Util.AskYesOrNoQuestion(f"The Serial Number is usually 15 letters or numbers, you have entered {len(printerSn)}. Do you want to try again? "): + continue + + retryEntry = False + for c in printerSn: + if not c.isdigit() and not c.isalpha(): + if Util.AskYesOrNoQuestion("The Serial Number should only be letters and numbers, you seem to have entered something else. Do you want to try again? "): + retryEntry = True + break + if retryEntry: + continue + + # Accept the input. + break + + # Scan for the local IP subset for possible matches. + Logger.Blank() + Logger.Blank() + Logger.Warn("Searching for your Bambu printer on your network, this will take about 5 seconds...") + ips = NetworkSearch.ScanForInstances_Bambu(Logger.GetPyLogger(), accessCode, printerSn) + + Logger.Blank() + Logger.Blank() + + # There should only be one IP found or none, because there should be no other printer that matches the same access code and printer serial number. + if len(ips) == 1: + ip = ips[0] + return (ip, NetworkSearch.c_BambuDefaultPortStr, accessCode, printerSn) + + Logger.Blank() + Logger.Blank() + Logger.Blank() + Logger.Error("We were unable to automatically find your printer on your network using these details:") + Logger.Info(f" Access Code: {accessCode}") + Logger.Info(f" Serial Number: {printerSn}") + Logger.Blank() + Logger.Header("Make sure your printer is on a full booted and verify the values above are correct.") + Logger.Blank() + if not Util.AskYesOrNoQuestion("Are the Access Code and Serial Number correct?"): + # Loop back to the very top, to restart the entire setup, allowing the user to enter their values again. + Logger.Blank() + Logger.Blank() + Logger.Blank() + Logger.Blank() + continue - # Setup the callback functions - def OnOpened(ws): - Logger.Debug(f"Test [{url}] - WS Opened") - def OnMsg(ws, msg): - with lock: - if "success" in result: - return + # Enter manual IP setup mode + while True: + Logger.Blank() + Logger.Blank() + Logger.Info("Since we can't automatically find your printer, we can get the IP address manually.") + Logger.Info("You Bambu printer's IP address can be found using the screen on your printer, in the Networking settings.") + Logger.Blank() + Logger.Warn("Follow this link for a step-by-step guide to find the IP address of your printer:") + Logger.Warn("https://octoeverywhere.com/s/bambu-ip") + Logger.Blank() + ip = input("Enter your printer's IP Address: ") + ip = ip.strip() + Logger.Blank() + Logger.Info("Trying to connect to your printer...") + result = NetworkSearch.ValidateConnection_Bambu(Logger.GetPyLogger(), ip, accessCode, printerSn, timeoutSec=10.0) + Logger.Blank() + Logger.Blank() + if result.Success(): + return (ip, NetworkSearch.c_BambuDefaultPortStr, accessCode, printerSn) + if result.FailedToConnect: + Logger.Error("Failed to connect to your Bambu printer, ensure the IP address is correct and the printer is connected to the network.") + elif result.FailedAuth: + Logger.Error("Failed to connect to your Bambu printer, the Access Code was incorrect.") + _ = input("Press any key to continue.") + # Breaking this loop will return us to the main setup loop, which will do the entire access code and sn entry again. + break + elif result.FailedSerialNumber: + Logger.Error("Failed to connect to your Bambu printer, the Serial Number was incorrect.") + _ = input("Press any key to continue.") + # Breaking this loop will return us to the main setup loop, which will do the entire access code and sn entry again. + break + else: + Logger.Error("Failed to connect to your Bambu printer.") + + # If we got here, the IP address is wrong or something else. + Logger.Blank() + Logger.Info("Pick one of the following:") + Logger.Info(" 1) Enter the IP address again.") + Logger.Info(" 2) Enter your Access Code and Serial Number.") + Logger.Blank() + c = input("Pick one or two: ") try: - # Try to see if the message looks like one of the first moonraker messages. - msgStr = msg.decode('utf-8') - Logger.Debug(f"Test [{url}] - WS message `{msgStr}`") - if "moonraker" in msgStr.lower(): - Logger.Debug(f"Test [{url}] - Found Moonraker message, success!") - result["success"] = True - doneEvent.set() + cInt = int(c.strip()) + if cInt == 1: + # Restart IP entry. + continue + else: + # Breaking this loop will return us to the main setup loop, which will do the entire access code and sn entry again. + break except Exception: - pass - def OnClosed(ws): - Logger.Debug(f"Test [{url}] - Closed") - doneEvent.set() - def OnError(ws, exception): - Logger.Debug(f"Test [{url}] - Error: {str(exception)}") - with lock: - result["exception"] = exception - doneEvent.set() - - # Create the websocket - Logger.Debug(f"Checking for moonraker using the address: `{url}`") - ws = Client(url, onWsOpen=OnOpened, onWsMsg=OnMsg, onWsError=OnError, onWsClose=OnClosed) - ws.RunAsync() - - # Wait for the event or a timeout. - doneEvent.wait(timeoutSec) - - # Get the results before we close. - capturedSuccess = False - capturedEx = None - with lock: - if result.get("success", None) is not None: - capturedSuccess = result["success"] - if result.get("exception", None) is not None: - capturedEx = result["exception"] - - # Ensure the ws is closed - try: - ws.Close() - except Exception: - pass - - return (capturedSuccess, capturedEx) - - - # Scans the subnet for Moonraker instances. - # Returns a list of IPs where moonraker was found. - def _ScanForMoonrakerInstances(self): - foundIps = [] - try: - localIp = self._TryToGetLocalIp() - if localIp is None or len(localIp) == 0: - Logger.Debug("Failed to get local IP") - return foundIps - Logger.Debug(f"Local IP found as: {localIp}") - if ":" in localIp: - Logger.Info("IPv6 addresses aren't supported for local discovery.") - return foundIps - lastDot = localIp.rfind(".") - if lastDot == -1: - Logger.Info("Failed to find last dot in local IP?") - return foundIps - ipPrefix = localIp[:lastDot+1] - - counter = 0 - doneThreads = [0] - totalThreads = 255 - threadLock = threading.Lock() - doneEvent = threading.Event() - while counter <= totalThreads: - fullIp = ipPrefix + str(counter) - def threadFunc(ip): - try: - success, _ = self._CheckForMoonraker(ip, "7125", 5.0) - with threadLock: - if success: - foundIps.append(ip) - doneThreads[0] += 1 - if doneThreads[0] == totalThreads: - doneEvent.set() - except Exception as e: - Logger.Error(f"Moonraker scan failed for {ip} "+str(e)) - t = threading.Thread(target=threadFunc, args=[fullIp]) - t.start() - counter += 1 - doneEvent.wait() - return foundIps - except Exception as e: - Logger.Error("Failed to scan for Moonraker instances. "+str(e)) - return foundIps - - - def _TryToGetLocalIp(self) -> str: - # Find the local IP. Works on Windows and Linux. Always gets the correct routable IP. - # https://stackoverflow.com/a/28950776 - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - ip = None - try: - # doesn't even have to be reachable - s.connect(('1.1.1.1', 1)) - ip = s.getsockname()[0] - except Exception: - pass - finally: - s.close() - return ip + # Default to a full restart. + # Breaking this loop will return us to the main setup loop, which will do the entire access code and sn entry again. + break diff --git a/py_installer/Permissions.py b/py_installer/Permissions.py index 6407065..f1fc6d3 100644 --- a/py_installer/Permissions.py +++ b/py_installer/Permissions.py @@ -38,7 +38,7 @@ def EnsureRunningAsRootOrSudo(self, context:Context) -> None: # But regardless of the user, we must have sudo permissions. # pylint: disable=no-member # Linux only if os.geteuid() != 0: - if context.Debug: + if context.SkipSudoActions: Logger.Warn("Not running as root, but ignoring since we are in debug.") else: raise Exception("Script not ran as root or using sudo. This is required to integrate into Moonraker.") diff --git a/py_installer/Service.py b/py_installer/Service.py index 85548a9..f85552b 100644 --- a/py_installer/Service.py +++ b/py_installer/Service.py @@ -60,17 +60,19 @@ def Install(self, context:Context): # Install for debian setups def _InstallDebian(self, context:Context, argsJsonBase64:str, moduleNameToRun:str): + serviceName = "Bambu Labs Printers" if context.IsBambuSetup else "Moonraker" + optionalAfter = "" if context.IsBambuSetup else "moonraker.service" s = f'''\ - # OctoEverywhere For Moonraker Service + # OctoEverywhere For {serviceName} [Unit] - Description=OctoEverywhere For Moonraker - # Start after network and moonraker has started. - After=network-online.target moonraker.service + Description=OctoEverywhere For {serviceName} + # Start after network. + After=network-online.target {optionalAfter} [Install] WantedBy=multi-user.target - # Simple service, targeting the user that was used to install the service, simply running our moonraker py host script. + # Simple service, targeting the user that was used to install the service, simply running our py host script. [Service] Type=simple User={context.UserName} diff --git a/setup.py b/setup.py index 77cdf10..805b2c0 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this is also parsed by the moonraker module to pull the version, so the string and format must remain the same! -plugin_version = "2.11.0" +plugin_version = "2.12.0" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 41c32c9fa717b4cf15746ea6aade8b0f628d1341 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Tue, 5 Mar 2024 21:41:03 -0800 Subject: [PATCH 036/328] Fixing an updater bug with the permssions class. --- py_installer/ConfigHelper.py | 4 +++- py_installer/Permissions.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/py_installer/ConfigHelper.py b/py_installer/ConfigHelper.py index 4737c82..43d631b 100644 --- a/py_installer/ConfigHelper.py +++ b/py_installer/ConfigHelper.py @@ -143,11 +143,13 @@ def DoesConfigFileExist(context:Context = None, configFolderPath:str = None) -> # Given a context or config file path, this returns file path of the config. + # If the context is missing the ConfigFolder, None is returned. @staticmethod def GetConfigFilePath(context:Context = None, configFolderPath:str = None): if context is not None: if context.ConfigFolder is None: - raise Exception("GetConfigFilePath context doesn't have a ConfigFolder string.") + # Don't throw here, return None and let the caller handle it, incase it's ok to not have a config folder set. + return None return Config.GetConfigFilePath(context.ConfigFolder) if configFolderPath is not None: return Config.GetConfigFilePath(configFolderPath) diff --git a/py_installer/Permissions.py b/py_installer/Permissions.py index f1fc6d3..e80ba93 100644 --- a/py_installer/Permissions.py +++ b/py_installer/Permissions.py @@ -56,6 +56,7 @@ def EnsureFinalPermissions(self, context:Context): # We try to set permissions to all paths and files in the context, some might be null # due to the setup mode. We don't care to difference the setup mode here, because the context # validation will do that for us already. Thus if a field is None, its ok. + # NOTE - In update mode, most of the paths in the context are None, since it's updating all plugins at once, which is fine. def SetPermissions(path:str): if path is not None and len(path) != 0: Util.SetFileOwnerRecursive(path, context.UserName) From b38e8daffdd311c94cb07ac11e8ad67bfdb44420 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Tue, 5 Mar 2024 22:13:14 -0800 Subject: [PATCH 037/328] Fixing bug in the octohttp result class --- octoeverywhere/octohttprequest.py | 7 ++----- setup.py | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/octoeverywhere/octohttprequest.py b/octoeverywhere/octohttprequest.py index 4713617..198e429 100644 --- a/octoeverywhere/octohttprequest.py +++ b/octoeverywhere/octohttprequest.py @@ -76,7 +76,8 @@ def GetPathType(url): # customBodyStreamClosedCallback() -> None : MUST BE CALLED when this Result object is closed, to clean up the stream. class Result(): def __init__(self, statusCode:int, headers:dict, url:str, didFallback:bool, fullBodyBuffer=None, requestLibResponseObj:requests.Response=None, customBodyStreamCallback=None, customBodyStreamClosedCallback=None): - self._statusCode = statusCode + # Status code isn't a property because some things need to set it externally to the class. (Result.StatusCode = 302) + self.StatusCode = statusCode self._headers = headers self._url:str = url self._requestLibResponseObj = requestLibResponseObj @@ -90,10 +91,6 @@ def __init__(self, statusCode:int, headers:dict, url:str, didFallback:bool, full if (self._customBodyStreamCallback is not None and self._customBodyStreamClosedCallback is None) or (self._customBodyStreamCallback is None and self._customBodyStreamClosedCallback is not None): raise Exception("Both the customBodyStreamCallback and customBodyStreamClosedCallback must be set!") - @property - def StatusCode(self) -> int: - return self._statusCode - @property def Headers(self) -> dict: return self._headers diff --git a/setup.py b/setup.py index 805b2c0..1764a4e 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this is also parsed by the moonraker module to pull the version, so the string and format must remain the same! -plugin_version = "2.12.0" +plugin_version = "2.12.1" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From e4fcfc336ab36762863fdf44a548ea5ce14f1af1 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Thu, 7 Mar 2024 20:49:42 -0800 Subject: [PATCH 038/328] Plugin Version 3.0 - With all of the final Bambu Connect logic!!!! --- .vscode/settings.json | 4 + bambu_octoeverywhere/bambuclient.py | 8 +- bambu_octoeverywhere/bambucommandhandler.py | 36 ++- bambu_octoeverywhere/bambuhost.py | 6 +- bambu_octoeverywhere/bambumodels.py | 59 ++++ bambu_octoeverywhere/bambustatetranslater.py | 190 ++++++------ install.sh | 6 +- moonraker_octoeverywhere/moonrakerclient.py | 19 +- .../moonrakercommandhandler.py | 1 + moonraker_octoeverywhere/moonrakerhost.py | 6 +- octoeverywhere/notificationshandler.py | 275 +++++++++++------- octoeverywhere/printinfo.py | 235 +++++++++++++++ octoprint_octoeverywhere/__init__.py | 11 +- octoprint_octoeverywhere/__main__.py | 4 + .../octoprintcommandhandler.py | 6 +- setup.py | 4 +- 16 files changed, 641 insertions(+), 229 deletions(-) create mode 100644 octoeverywhere/printinfo.py diff --git a/.vscode/settings.json b/.vscode/settings.json index 57ac3cc..5b1609e 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -78,6 +78,7 @@ "journalctl", "JRPC", "jsonify", + "kbytes", "keepalive", "keyvalidator", "KIAUH", @@ -110,6 +111,7 @@ "networksearch", "noatuoselect", "notificationshandler", + "ntpsec", "Ocoto", "ocotomessage", "Octo", @@ -151,6 +153,8 @@ "printernotifications", "printerprofiles", "printerstateobject", + "printinfo", + "printkeeper", "PROCD", "Proto", "proxying", diff --git a/bambu_octoeverywhere/bambuclient.py b/bambu_octoeverywhere/bambuclient.py index a149518..53b5399 100644 --- a/bambu_octoeverywhere/bambuclient.py +++ b/bambu_octoeverywhere/bambuclient.py @@ -93,6 +93,9 @@ def _ClientWorker(self): while True: ipOrHostname = None try: + # Before we try to connect, ensure we tell the state translator that we are starting a new connection. + self.StateTranslator.ResetForNewConnection() + # We always connect locally. We use encryption, but the printer doesn't have a trusted # cert root, so we have to disable the cert root checks. self.Client = mqtt.Client(mqtt.CallbackAPIVersion.VERSION2) @@ -260,9 +263,10 @@ def _OnMessage(self, client, userdata, msg:mqtt.MQTTMessage): # Send all messages to the state translator # This must happen AFTER we update the State object, so it's current. - # We also pass the State object since we know it's not None try: - self.StateTranslator.OnMqttMessage(msg, self.State, isFirstFullSyncResponse) + # Only send the message along if there's a state. This can happen if a push_status isn't the first message we receive. + if self.State is not None: + self.StateTranslator.OnMqttMessage(msg, self.State, isFirstFullSyncResponse) except Exception as e: Sentry.Exception("Exception calling StateTranslator.OnMqttMessage", e) diff --git a/bambu_octoeverywhere/bambucommandhandler.py b/bambu_octoeverywhere/bambucommandhandler.py index 49f0652..22c0c60 100644 --- a/bambu_octoeverywhere/bambucommandhandler.py +++ b/bambu_octoeverywhere/bambucommandhandler.py @@ -1,6 +1,8 @@ from octoeverywhere.commandhandler import CommandResponse +from octoeverywhere.printinfo import PrintInfoManager from .bambuclient import BambuClient +from .bambumodels import BambuPrintErrors # This class implements the Platform Command Handler Interface class BambuCommandHandler: @@ -29,10 +31,21 @@ def GetCurrentJobStatus(self): return None # Map the state - # TODO - Add "error" if possible # Possible states: https://github.com/greghesp/ha-bambulab/blob/e72e343acd3279c9bccba510f94bf0e291fe5aaa/custom_components/bambu_lab/pybambu/const.py#L83C1-L83C21 state = "idle" - if bambuState.gcode_state is not None: + errorStr_CanBeNone = None + + # Before checking the state, see if the print is in an error state. + # This error state can be common among other states, like "IDLE" or "PAUSE" + printError = bambuState.GetPrinterError() + if printError is not None: + # Always set the state to error. + # If we can match a known state, return a good string that can be shown for the user. + state = "error" + if printError == BambuPrintErrors.FilamentRunOut: + errorStr_CanBeNone = "Filament Run Out" + # If we aren't in error, use the state + elif bambuState.gcode_state is not None: gcodeState = bambuState.gcode_state if gcodeState == "IDLE" or gcodeState == "INIT" or gcodeState == "OFFLINE" or gcodeState == "UNKNOWN": state = "idle" @@ -61,8 +74,6 @@ def GetCurrentJobStatus(self): else: self.Logger.warn(f"Unknown gcode_state state in print state: {gcodeState}") - # TODO - If in an error state, set some context as to why. - errorStr_CanBeNone = None # Get current layer info # None = The platform doesn't provide it. @@ -75,13 +86,18 @@ def GetCurrentJobStatus(self): if bambuState.total_layer_num is not None: totalLayersInt = int(bambuState.total_layer_num) - # Get duration and filename. + # Get the filename. + fileName = bambuState.GetFileNameWithNoExtension() + if fileName is None: + fileName = "" + + # For Bambu, the printer doesn't report the duration or the print start time. + # Thus we have to track it ourselves in our print info. + # When the print is over, a final print duration is set, so this doesn't keep going from print start. durationSec = 0 - fileName = "" - if bambuState.gcode_file is not None: - fileName = bambuState.gcode_file - #if "gcode_file" in res: - #durationSec = res["gcode_file"] + pi = PrintInfoManager.Get().GetPrintInfo(bambuState.GetPrintCookie()) + if pi is not None: + durationSec = pi.GetPrintDurationSec() # If we have a file name, try to get the current filament usage. filamentUsageMm = 0 diff --git a/bambu_octoeverywhere/bambuhost.py b/bambu_octoeverywhere/bambuhost.py index fa07ecf..c5e3636 100644 --- a/bambu_octoeverywhere/bambuhost.py +++ b/bambu_octoeverywhere/bambuhost.py @@ -3,8 +3,9 @@ from octoeverywhere.mdns import MDns from octoeverywhere.sentry import Sentry -from octoeverywhere.hostcommon import HostCommon from octoeverywhere.telemetry import Telemetry +from octoeverywhere.hostcommon import HostCommon +from octoeverywhere.printinfo import PrintInfoManager from octoeverywhere.webcamhelper import WebcamHelper from octoeverywhere.octopingpong import OctoPingPong from octoeverywhere.commandhandler import CommandHandler @@ -91,6 +92,9 @@ def RunBlocking(self, configPath, localStorageDir, repoRoot, devConfig_CanBeNone # Init the mdns client MDns.Init(self.Logger, localStorageDir) + # Setup the print info manager. + PrintInfoManager.Init(self.Logger, localStorageDir) + # For bambu, there's no frontend to connect to, so we disable the http relay system. OctoHttpRequest.SetDisableHttpRelay(True) diff --git a/bambu_octoeverywhere/bambumodels.py b/bambu_octoeverywhere/bambumodels.py index 9c7cc46..0930fa5 100644 --- a/bambu_octoeverywhere/bambumodels.py +++ b/bambu_octoeverywhere/bambumodels.py @@ -2,6 +2,15 @@ import logging from enum import Enum + +# Known printer error types. +# Note that the print state doesn't have to be ERROR to have an error, during a print it's "PAUSED" but the print_error value is not 0. +# Here's the full list https://e.bambulab.com/query.php?lang=en +class BambuPrintErrors(Enum): + Unknown = 1 # This will be most errors, since most of them aren't mapped + FilamentRunOut = 2 + + # Since MQTT syncs a full state and then sends partial updates, we keep track of the full state # and then apply updates on top of it. We basically keep a locally cached version of the state around. class BambuState: @@ -20,6 +29,8 @@ def __init__(self) -> None: self.bed_temper:int = None self.bed_target_temper:int = None self.mc_remaining_time:int = None + self.project_id:str = None + self.print_error:int = None # Custom fields self.LastTimeRemainingWallClock:float = None @@ -33,11 +44,13 @@ def OnUpdate(self, msg:dict) -> None: self.layer_num = msg.get("layer_num", self.layer_num) self.total_layer_num = msg.get("total_layer_num", self.total_layer_num) self.gcode_file = msg.get("gcode_file", self.gcode_file) + self.project_id = msg.get("project_id", self.project_id) self.mc_percent = msg.get("mc_percent", self.mc_percent) self.nozzle_temper = msg.get("nozzle_temper", self.nozzle_temper) self.nozzle_target_temper = msg.get("nozzle_target_temper", self.nozzle_target_temper) self.bed_temper = msg.get("bed_temper", self.bed_temper) self.bed_target_temper = msg.get("bed_target_temper", self.bed_target_temper) + self.print_error = msg.get("print_error", self.print_error) # Time remaining has some custom logic, so as it's queried each time it keep counting down in seconds, since Bambu only gives us minutes. old_mc_remaining_time = self.mc_remaining_time @@ -77,6 +90,47 @@ def IsPaused(self) -> bool: return self.gcode_state == "PAUSE" + # If there is a file name, this returns it without the final . + def GetFileNameWithNoExtension(self): + if self.gcode_file is None: + return None + pos = self.gcode_file.rfind(".") + if pos == -1: + return self.gcode_file + return self.gcode_file[:pos] + + + # Returns a unique string for this print. + # This string should be as unique as possible, but always the same for the same print. + # See details in NotificationHandler._RecoverOrRestForNewPrint + def GetPrintCookie(self) -> str: + # From testing, the project_id is always unique for cloud based prints, but is 0 for local prints. + # The file name changes most of the time, so the combination of both makes a good pair. + return f"{self.project_id}-{self.GetFileNameWithNoExtension()}" + + + # If the printer is in an error state, this tries to return the type, if known. + # If the printer is not in an error state, None is returned. + def GetPrinterError(self) -> BambuPrintErrors: + # If there is a printer error, this is not 0 + if self.print_error is None or self.print_error == 0: + return None + # There's a full list of errors here, we only care about some of them + # https://e.bambulab.com/query.php?lang=en + # We format the error into a hex the same way the are on the page, to make it easier. + # NOTE SOME ERRORS HAVE MULTPLE VALUES, SO GET THEM ALL! + # They have different values for the different AMS slots + h = hex(self.print_error)[2:].rjust(8, '0') + errorMap = { + "07008011": BambuPrintErrors.FilamentRunOut, + "07018011": BambuPrintErrors.FilamentRunOut, + "07028011": BambuPrintErrors.FilamentRunOut, + "07038011": BambuPrintErrors.FilamentRunOut, + "07FF8011": BambuPrintErrors.FilamentRunOut, + } + return errorMap.get(h, BambuPrintErrors.Unknown) + + # Different types of hardware. class BambuPrinters(Enum): Unknown = 1 @@ -99,6 +153,7 @@ class BambuVersion: def __init__(self, logger:logging.Logger) -> None: self.Logger = logger + self.HasLoggedPrinterVersion = False # We only parse out what we currently use. self.SoftwareVersion:str = None self.HardwareVersion:str = None @@ -156,3 +211,7 @@ def OnUpdate(self, msg:dict) -> None: if self.PrinterName is None or self.PrinterName is BambuPrinters.Unknown: self.Logger.warn(f"Unknown printer type. CPU:{self.Cpu}, Project Name: {self.ProjectName}, Hardware Version: {self.HardwareVersion}") self.PrinterName = BambuPrinters.Unknown + + if self.HasLoggedPrinterVersion is False: + self.HasLoggedPrinterVersion = True + self.Logger.info(f"Printer Version: {self.PrinterName}, CPU: {self.Cpu}, Project: {self.ProjectName} Hardware: {self.HardwareVersion}, Software: {self.SoftwareVersion}, Serial: {self.SerialNumber}") diff --git a/bambu_octoeverywhere/bambustatetranslater.py b/bambu_octoeverywhere/bambustatetranslater.py index 475095f..73408a1 100644 --- a/bambu_octoeverywhere/bambustatetranslater.py +++ b/bambu_octoeverywhere/bambustatetranslater.py @@ -1,7 +1,10 @@ +import time + from octoeverywhere.notificationshandler import NotificationsHandler +from octoeverywhere.printinfo import PrintInfoManager from .bambuclient import BambuClient -from .bambumodels import BambuState +from .bambumodels import BambuState, BambuPrintErrors # This class is responsible for listening to the mqtt messages to fire off notifications # and to act as the printer state interface for Bambu printers. @@ -10,17 +13,20 @@ class BambuStateTranslator: def __init__(self, logger) -> None: self.Logger = logger self.NotificationsHandler:NotificationsHandler = None - self.HasPendingPrintStart = False - self.HasPendingPrintPause = False - self.HasPendingPrintResume = False - self.HasPendingPrintFailed = False - self.WasInRunningStateLastUpdate = False + self.LastState:str = None def SetNotificationHandler(self, notificationHandler:NotificationsHandler): self.NotificationsHandler = notificationHandler + # Called by the client just before it tires to make a new connection. + # This is used to let us know that we are in an unknown state again, until we can re-sync. + def ResetForNewConnection(self): + # Reset the last state to indicate that we don't know what it is. + self.LastState = None + + # Fired when any mqtt message comes in. # State will always be NOT NONE, since it's going to be created before this call. # The isFirstFullSyncResponse flag indicates if this is the first full state sync of a new connection. @@ -28,107 +34,102 @@ def OnMqttMessage(self, msg:dict, bambuState:BambuState, isFirstFullSyncResponse # First, if we have a new connection and we just synced, make sure the notification handler is in sync. if isFirstFullSyncResponse: - # TODO - Ideally we pass the full print duration. - self.NotificationsHandler.OnRestorePrintIfNeeded(bambuState.IsPrinting(False), bambuState.IsPaused(), bambuState.gcode_file, None) - self.WasInRunningStateLastUpdate = False - - # Remember that each delta could have multiple pieces of needed information in them - # And that we will only get the delta updates once! + self.NotificationsHandler.OnRestorePrintIfNeeded(bambuState.IsPrinting(False), bambuState.IsPaused(), bambuState.GetPrintCookie()) + + # Bambu does send some commands when actions happen, but they don't always get sent for all state changes. + # For example, if a user issues a pause command, we see the command. But if the print goes into an error an pauses, we don't get a pause command. + # Thus, we have to rely on keeping track of that state and knowing when it changes. + # Note we check state for all messages, not just push_status, but it doesn't matter because it will only change on push_status anyways. + # Here's a list of all states: https://github.com/greghesp/ha-bambulab/blob/e72e343acd3279c9bccba510f94bf0e291fe5aaa/custom_components/bambu_lab/pybambu/const.py#L83C1-L83C21 + if self.LastState != bambuState.gcode_state: + # We know the state changed. + if self.LastState is None: + # If the last state is None, this is mostly likely the first time we've seen a state. + # All we want to do here is update last state to the new state. + pass + # Check if we are now in a printing state we use the common function so the definition of "printing" stays common. + elif bambuState.IsPrinting(False): + if self.LastState == "PAUSE": + self.BambuOnResume(bambuState) + else: + self.BambuOnStart(bambuState) + # Check for the paused state + elif bambuState.IsPaused(): + # If the error is temporary, like a filament run out, the printer goes into a paused state + # with the printer_error set. + self.BambuOnPauseOrTempError(bambuState) + # Check for the print ending in failure (like if the user stops it by command) + elif bambuState.gcode_state == "FAILED": + self.BambuOnFailed(bambuState) + # Check for a successful print ending. + elif bambuState.gcode_state == "FINISH": + self.BambuOnComplete(bambuState) + + # Always capture the new state. + self.LastState = bambuState.gcode_state # - # Next, handle any explicit onetime command actions. - if "print" in msg: - if "command" in msg["print"]: - # Note about commands. Since these commands come before the push_status update - # The State object MIGHT NOT BE UPDATED TO THE CORRECT STATE WHEN THEY FIRE. - # For example, the gcode_state will be the last state when project_file fires, because it hasn't updated yet. - # That can be really bad, like for ShouldPrintingTimersBeRunning, which will then return an incorrect value. - # Thus we defer the command action until we see the next push_status come in. - command = msg["print"]["command"] - if command == "project_file": - self.HasPendingPrintStart = True - elif command == "pause": - self.HasPendingPrintPause = True - elif command == "resume": - self.HasPendingPrintResume = True - elif command == "stop": - # This is a stop, I think only user generated? - # TODO - will this fire for other types or errors? - self.HasPendingPrintFailed = True - - # If we have a status update, we know our State should be current, so fire any deferred commands. - elif command == "push_status": - if self.HasPendingPrintStart: - # We have to be really careful with this notification, because it kicks off a lot of things. - # We have to wait until the State is reporting RUNNING before we send it, to ensure things like - # ShouldPrintingTimersBeRunning are in a good state when all of the new things query them. - if bambuState.gcode_state is not None and bambuState.gcode_state == "RUNNING": - self.HasPendingPrintStart = False - self.BambuOnStart(bambuState) - else: - self.Logger.info("Deferring print start until the gcode_state is running...") - - if self.HasPendingPrintPause: - self.HasPendingPrintPause = False - self.BambuOnPause(bambuState) - - if self.HasPendingPrintResume: - self.HasPendingPrintResume = False - self.BambuOnResume(bambuState) - - if self.HasPendingPrintFailed: - self.HasPendingPrintFailed = False - self.BambuOnFailed(bambuState) - - # - # Next - Handle notifications that aren't based off one time events. - # - # These are harder to get right, because the printer will send full state objects sometimes when IDLE or PRINTING. - # Thus if we respond to them, it might not be the correct time. For example, the full sync will always include mc_percent, but we - # don't want to fire BambuOnPrintProgress if we aren't printing. - # - # We only want to consider firing these events if we know this isn't the first time sync from a new connection - # and we are currently tacking a print. - if not isFirstFullSyncResponse and self.NotificationsHandler.IsTrackingPrint(): - # Percentage progress update - if "mc_percent" in msg["print"]: - self.BambuOnPrintProgress(bambuState) - - # Complete is hard, because there's no explicitly one time command for print success. - # We also don't want to rely on IsTrackingPrint, because there's a small window where the state could be updated - # and one of the notification threads could check ShouldPrintingTimersBeRunning, it be False, and stop them. - # So, we keep track of if the state was RUNNING and then goes to FINISHED - if bambuState.gcode_state is not None and bambuState.gcode_state == "FINISH": - if self.WasInRunningStateLastUpdate: - # The last state was running and now it's FINISHED, the print is complete. - self.BambuOnComplete(bambuState) - - # Always update the flag. - self.WasInRunningStateLastUpdate = bambuState.gcode_state is not None and bambuState.gcode_state == "RUNNING" + # Next - Handle the progress update. + # + # These are harder to get right, because the printer will send full state objects sometimes when IDLE or PRINTING. + # Thus if we respond to them, it might not be the correct time. For example, the full sync will always include mc_percent, but we + # don't want to fire BambuOnPrintProgress if we aren't printing. + # + # We only want to consider firing these events if we know this isn't the first time sync from a new connection + # and we are currently tacking a print. + if not isFirstFullSyncResponse and self.NotificationsHandler.IsTrackingPrint(): + # Percentage progress update + if "mc_percent" in msg["print"]: + self.BambuOnPrintProgress(bambuState) + + # Since bambu doesn't tell us a print duration, we need to figure out when it ends ourselves. + # This is different from the state changes above, because if we are ever not printing for any reason, + # We want to finalize any current print. + if bambuState.IsPrinting(True) is False: + # See if there's a print info for the last print. + pi = PrintInfoManager.Get().GetPrintInfo(bambuState.GetPrintCookie()) + if pi is not None: + # Check if the print info has a final duration set yet or not. + if pi.GetFinalPrintDurationSec() is None: + # We know we aren't printing, so regardless of the non-printing state, set the final duration. + pi.SetFinalPrintDurationSec(int(time.time()-pi.GetLocalPrintStartTimeSec())) def BambuOnStart(self, bambuState:BambuState): - # We can only get the file name from Bambu. - self.NotificationsHandler.OnStarted(self._GetFileNameOrNone(bambuState), 0, 0) + # We must pass the unique cookie name for this print and any other details we can. + self.NotificationsHandler.OnStarted(bambuState.GetPrintCookie(), bambuState.GetFileNameWithNoExtension()) def BambuOnComplete(self, bambuState:BambuState): # We can only get the file name from Bambu. - self.NotificationsHandler.OnDone(self._GetFileNameOrNone(bambuState), None) + self.NotificationsHandler.OnDone(bambuState.GetFileNameWithNoExtension(), None) - def BambuOnPause(self, bambuState:BambuState): - self.NotificationsHandler.OnPaused(self._GetFileNameOrNone(bambuState)) + def BambuOnPauseOrTempError(self, bambuState:BambuState): + # For errors that are user fixable, like filament run outs, the printer will go into a paused state with + # a printer error message. In this case we want to fire different things. + err = bambuState.GetPrinterError() + if err is None: + # If error is none, this is a user pause + self.NotificationsHandler.OnPaused(bambuState.GetFileNameWithNoExtension()) + return + # Otherwise, try to match the error. + if err == BambuPrintErrors.FilamentRunOut: + self.NotificationsHandler.OnFilamentChange() + return + + # Send a generic error. + self.NotificationsHandler.OnUserInteractionNeeded() def BambuOnResume(self, bambuState:BambuState): - self.NotificationsHandler.OnResume(self._GetFileNameOrNone(bambuState)) + self.NotificationsHandler.OnResume(bambuState.GetFileNameWithNoExtension()) def BambuOnFailed(self, bambuState:BambuState): # TODO - Right now this is only called by what we think are use requested cancels. # How can we add this for print stopping errors as well? - self.NotificationsHandler.OnFailed(self._GetFileNameOrNone(bambuState), None, "cancelled") + self.NotificationsHandler.OnFailed(bambuState.GetFileNameWithNoExtension(), None, "cancelled") def BambuOnPrintProgress(self, bambuState:BambuState): @@ -137,22 +138,9 @@ def BambuOnPrintProgress(self, bambuState:BambuState): self.NotificationsHandler.OnPrintProgress(None, float(bambuState.mc_percent)) # TODO - Handlers - # # # Fired when OctoPrint or the printer hits an error. # def OnError(self, error): - # # Fired when the waiting command is received from the printer. - # def OnWaiting(self): - - # # Fired when we get a M600 command from the printer to change the filament - # def OnFilamentChange(self): - - # # Fired when the printer needs user interaction to continue - # def OnUserInteractionNeeded(self): - - - def _GetFileNameOrNone(self, bambuState:BambuState) -> str: - return bambuState.gcode_file # # diff --git a/install.sh b/install.sh index bffb0b2..a7a46e7 100755 --- a/install.sh +++ b/install.sh @@ -325,14 +325,12 @@ cat << EOF @@@@@@@@@@@@@@@,,,,,,,,,,,,,,,,,,,,,@@@@@@@@@@@@@@ EOF log_blank -#log_header " OctoEverywhere For Klipper And Bambu Labs" -log_important " OctoEverywhere For Klipper" +log_header " OctoEverywhere For Klipper And Bambu Labs" log_blue " The 3D Printing Communities #1 Remote Access And AI Cloud Service" log_blank log_blank log_important "OctoEverywhere empowers the worldwide maker community with..." -#log_info " - Free & Unlimited Mainsail, Fluidd, And Bambu Labs Printers Remote Access" -log_info " - Free & Unlimited Mainsail and Fluidd Remote Access" +log_info " - Free & Unlimited Mainsail, Fluidd, And Bambu Labs Printers Remote Access" log_info " - Free & Unlimited Next-Gen AI Print Failure Detection" log_info " - Free Full Frame Rate & Full Resolution Webcam Streaming" log_info " - 5 Star Rated iOS & Android Apps" diff --git a/moonraker_octoeverywhere/moonrakerclient.py b/moonraker_octoeverywhere/moonrakerclient.py index 24ee0a1..cced740 100644 --- a/moonraker_octoeverywhere/moonrakerclient.py +++ b/moonraker_octoeverywhere/moonrakerclient.py @@ -834,7 +834,7 @@ def OnPrintStart(self, fileName): fileSizeKBytes = max(fileSizeKBytes, 0) # Fire on started. - self.NotificationHandler.OnStarted(fileName, fileSizeKBytes, filamentUsageMm) + self.NotificationHandler.OnStarted(self._GetPrintCookie(fileName), fileName, fileSizeKBytes, filamentUsageMm) def OnDone(self): @@ -1069,6 +1069,20 @@ def IsPrintWarmingUp(self): # Helpers # + # Returns a unique string for this print. + # This string should be as unique as possible, but always the same for the same print. + # See details in NotificationHandler._RecoverOrRestForNewPrint + def _GetPrintCookie(self, fileName:str) -> str: + # For Moonraker, there's no way to differentiate between prints beyond the basic things like the file name. + # This means that there is a possibility that the print cookie will match, on back to back prints. + # However on each start we will clear any Print info that exists, so it will clear each time. + # But if the service restarts mid print, we will still be able to recover it. + if fileName is None: + # If there is no filename, just use the time, which will make the print unrecoverable. + return f"{int(time.time())}" + return fileName + + def _InitPrintStateForFreshConnect(self): # Get the current state stats = self._GetCurrentPrintStats() @@ -1081,9 +1095,8 @@ def _InitPrintStateForFreshConnect(self): # the state as well as possible to get notifications in sync. state = stats["state"] fileName_CanBeNone = stats["filename"] - totalDurationFloatSec_CanBeNone = stats["total_duration"] # Use the total duration self.Logger.info("Printer state at socket connect is: "+state) - self.NotificationHandler.OnRestorePrintIfNeeded(state == "printing", state == "paused", fileName_CanBeNone, totalDurationFloatSec_CanBeNone) + self.NotificationHandler.OnRestorePrintIfNeeded(state == "printing", state == "paused", self._GetPrintCookie(fileName_CanBeNone)) # Queries moonraker for the current printer stats. diff --git a/moonraker_octoeverywhere/moonrakercommandhandler.py b/moonraker_octoeverywhere/moonrakercommandhandler.py index dd0cdb2..a0af6fc 100644 --- a/moonraker_octoeverywhere/moonrakercommandhandler.py +++ b/moonraker_octoeverywhere/moonrakercommandhandler.py @@ -72,6 +72,7 @@ def GetCurrentJobStatus(self): self.Logger.warn("MoonrakerCommandHandler failed to find the print_stats.status") # TODO - If in an error state, set some context as to why. + # This is shown to the user directly, so it must be short (think of a dashboard status) and formatted well. errorStr_CanBeNone = None # Get current layer info diff --git a/moonraker_octoeverywhere/moonrakerhost.py b/moonraker_octoeverywhere/moonrakerhost.py index 8c30b0a..93e8e20 100644 --- a/moonraker_octoeverywhere/moonrakerhost.py +++ b/moonraker_octoeverywhere/moonrakerhost.py @@ -3,10 +3,11 @@ from octoeverywhere.mdns import MDns from octoeverywhere.sentry import Sentry -from octoeverywhere.hostcommon import HostCommon from octoeverywhere.telemetry import Telemetry +from octoeverywhere.hostcommon import HostCommon from octoeverywhere.octopingpong import OctoPingPong from octoeverywhere.webcamhelper import WebcamHelper +from octoeverywhere.printinfo import PrintInfoManager from octoeverywhere.commandhandler import CommandHandler from octoeverywhere.octoeverywhereimpl import OctoEverywhere from octoeverywhere.octohttprequest import OctoHttpRequest @@ -115,6 +116,9 @@ def RunBlocking(self, klipperConfigDir, isCompanionMode, localStorageDir, servic # Allow the UI injector to run and do it's thing. UiInjector.Init(self.Logger, repoRoot) + # Setup the print info manager + PrintInfoManager.Init(self.Logger, localStorageDir) + # Setup the database helper self.MoonrakerDatabase = MoonrakerDatabase(self.Logger, printerId, pluginVersionStr) diff --git a/octoeverywhere/notificationshandler.py b/octoeverywhere/notificationshandler.py index c603e9b..28febf0 100644 --- a/octoeverywhere/notificationshandler.py +++ b/octoeverywhere/notificationshandler.py @@ -11,10 +11,11 @@ from .gadget import Gadget from .sentry import Sentry from .compat import Compat -from .snapshotresizeparams import SnapshotResizeParams +from .finalsnap import FinalSnap from .repeattimer import RepeatTimer from .webcamhelper import WebcamHelper -from .finalsnap import FinalSnap +from .printinfo import PrintInfoManager, PrintInfo +from .snapshotresizeparams import SnapshotResizeParams try: # On some systems this package will install but the import will fail due to a missing system .so. @@ -60,11 +61,8 @@ def __init__(self, logger:logging.Logger, printerStateInterface): self.FinalSnapObj:FinalSnap = None self.Gadget = Gadget(logger, self, self.PrinterStateInterface) - # Define all the vars - self.CurrentFileName = "" - self.CurrentFileSizeInKBytes = 0 - self.CurrentEstFilamentUsageMm = 0 - self.CurrentPrintStartTime = time.time() + # Define all the vars we use locally in the notification handler + self.PrintCookie = "" self.FallbackProgressInt = 0 self.MoonrakerReportedProgressFloat_CanBeNone = None self.PingTimerHoursReported = 0 @@ -77,22 +75,20 @@ def __init__(self, logger:logging.Logger, printerStateInterface): self.FirstLayerDoneSince = 0.0 self.ThirdLayerDoneSince = 0.0 self.ProgressCompletionReported = [] - self.PrintId = "none" - self.PrintStartTimeSec = 0 self.RestorePrintProgressPercentage = False self.SpammyEventTimeDict = {} self.SpammyEventLock = threading.Lock() - # Since all of the commands don't send things we need, we will also track them. - self.ResetForNewPrint(None) + # Call this to init all of the vars to their default values. + # But we pass none, so we don't delete any print infos that might be on disk we will try to recover when connected to the server. + self._RecoverOrRestForNewPrint(None) - def ResetForNewPrint(self, restoreDurationOffsetSec_OrNone): - self.CurrentFileName = "" - self.CurrentFileSizeInKBytes = 0 - self.CurrentEstFilamentUsageMm = 0 - self.CurrentPrintStartTime = time.time() + # Called to start a new print. + # On class init, this can be called with printCookie=None, but after that we should always have a print cookie. + def _RecoverOrRestForNewPrint(self, printCookie:str): + # We always reset these local notification handler values for new prints or recovered prints. self.FallbackProgressInt = 0 self.MoonrakerReportedProgressFloat_CanBeNone = None self.PingTimerHoursReported = 0 @@ -107,23 +103,6 @@ def ResetForNewPrint(self, restoreDurationOffsetSec_OrNone): self.zOffsetHasSeenPositiveExtrude = False self.RestorePrintProgressPercentage = False - # Ensure there's no final snap running. - self._getFinalSnapSnapshotAndStop() - - # If we have a restore time offset, back up the start time to make it reflect when the print started. - if restoreDurationOffsetSec_OrNone is not None: - self.CurrentPrintStartTime -= restoreDurationOffsetSec_OrNone - - # Each time a print starts, we generate a fixed length random id to identify it. - # This id is used to globally identify the print for the user, so it needs to have high entropy. - self.PrintId = ''.join(random.choices(string.ascii_uppercase + string.ascii_lowercase + string.digits, k=NotificationsHandler.PrintIdLength)) - - # Note the time this print started - self.PrintStartTimeSec = time.time() - - # Reset our anti spam times. - self._clearSpammyEventContexts() - # Build the progress completion reported list. # Add an entry for each progress we want to report, not including 0 and 100%. # This list must be in order, from the lowest value to the highest. @@ -139,6 +118,36 @@ def ResetForNewPrint(self, restoreDurationOffsetSec_OrNone): self.ProgressCompletionReported.append(ProgressCompletionReportItem(80.0, False)) self.ProgressCompletionReported.append(ProgressCompletionReportItem(90.0, False)) + # Reset our anti spam times. + self._clearSpammyEventContexts() + + # Ensure there's no final snap running. + self._getFinalSnapSnapshotAndStop() + + # The print cookie can only be None on class init. + # We pass None so we don't call the PrintInfoManager, which might create a new print info on disk. + # There might be a print info on disk we want to restore when the host connects to the printer. + if printCookie is None: + return + + # Always set the new print cookie + self.PrintCookie = printCookie + + # See if we have an existing print that matches this cookie on disk. + if PrintInfoManager.Get().GetPrintInfo(printCookie) is not None: + self.Logger.info(f"Print Manager recovered a print info from disk matching cookie: {printCookie}") + return + + # If we didn't find an existing print info, we need to make a new one. + + # Each time a print starts, we generate a fixed length random id to identify it. + # This id is used to globally identify the print for the user, so it needs to have high entropy. + printId = ''.join(random.choices(string.ascii_uppercase + string.ascii_lowercase + string.digits, k=NotificationsHandler.PrintIdLength)) + + # Always make a new print info for this new print. + # This is where we will store all of the vars for this print, and it's also written to disk if we need to recover the info. + PrintInfoManager.Get().CreateNewPrintInfo(printCookie, printId) + def SetPrinterId(self, printerId): self.PrinterId = printerId @@ -157,12 +166,26 @@ def SetGadgetServerProtocolAndDomain(self, protocolAndDomain): self.Gadget.SetServerProtocolAndDomain(protocolAndDomain) + # If there is an valid print cookie and we can get the info, this returns it. + # Returns None if there's no current print info. + def GetPrintInfo(self) -> PrintInfo: + if self.PrintCookie is None or len(self.PrintCookie) == 0: + return None + return PrintInfoManager.Get().GetPrintInfo(self.PrintCookie) + + def GetPrintId(self) -> str: - return self.PrintId + pi = self.GetPrintInfo() + if pi is None: + return None + return pi.GetPrintId() - def GetPrintStartTimeSec(self): - return self.PrintStartTimeSec + def GetPrintStartTimeSec(self) -> float: + pi = self.GetPrintInfo() + if pi is None: + return 0.0 + return pi.GetLocalPrintStartTimeSec() def GetGadget(self): @@ -188,43 +211,52 @@ def IsTrackingPrint(self) -> bool: # # Most importantly, we want to make sure the ping timer and thus Gadget get restored to the correct states. # - def OnRestorePrintIfNeeded(self, isPrinting:bool, isPaused:bool, fileName_CanBeNone, totalDurationFloatSec_CanBeNone): - if isPrinting: - # There is an active print. Check our state. - if self._IsPingTimerRunning(): - self.Logger.info("Restore client sync state: Detected an active print and our timers are already running, there's nothing to do.") - return - else: - self.Logger.info("Restore client sync state: Detected an active print but we aren't tracking it, so we will restore now.") - # We need to do the restore of a active print. - elif isPaused: - # There is a print currently paused, check to see if we have a filename, which indicates if we know of a print or not. - if self._HasCurrentPrintFileName(): - self.Logger.info("Restore client sync state: Detected a paused print, but we are already tracking a print, so there's nothing to do.") - return - else: - self.Logger.info("Restore client sync state: Detected a paused print, but we aren't tracking any prints, so we will restore now") - else: + def OnRestorePrintIfNeeded(self, isPrinting:bool, isPaused:bool, printCookie_CanBeNoneIfNoPrintIsActive:str = None): + + # First, check if there's no active print currently. + if (isPrinting is False and isPaused is False) or printCookie_CanBeNoneIfNoPrintIsActive is None: # There's no print running. if self._IsPingTimerRunning(): - self.Logger.info("Restore client sync state: Detected no active print but our ping timers ARE RUNNING. Stopping them now.") + self.Logger.info("Restore client sync state: There's no print running but the ping timers are running. Stopping them now.") self.StopTimers() return else: - self.Logger.info("Restore client sync state: Detected no active print and no ping timers are running, so there's nothing to do.") + self.Logger.info("Restore client sync state: There's no print and none of the timers are running.") return - # If we are here, we need to restore a print. - # The print can be in an active or paused state. - # Always restart for a new print. - # If totalDurationFloatSec_CanBeNone is not None, it will update the print start time to offset it correctly. - # This is important so our time elapsed number is correct. - self.ResetForNewPrint(totalDurationFloatSec_CanBeNone) + # Next, we know there's an active print, so check if we already are tracking it's print cookie. + # This is a scenario like the plugin didn't crash, but it lost the connection to the server, but it's back now. + printCookie = printCookie_CanBeNoneIfNoPrintIsActive + if self.PrintCookie is not None and self.PrintCookie == printCookie: + # We have a print cookie and the cookie matches. + # This means we just need to make sure the timer states are correct. + if isPrinting: + # There is an active print. Check our state. + if self._IsPingTimerRunning(): + self.Logger.info("Restore client sync state: We have the print cookie, detected an active print, and our timers are already running. So there's nothing to do.") + return + else: + self.Logger.info("Restore client sync state: We have a print cookie, detected and active print, but the timers aren't running, so we will start them now.") + self.StartPrintTimers(False, None) + return + # We need to restore so we start the print timers. + elif isPaused: + # The print is paused, check our state. + if self._IsPingTimerRunning(): + self.Logger.info("Restore client sync state: We have a print cookie, detected a paused print, but our ping timers ARE RUNNING. Stopping them now.") + self.StopTimers() + return + else: + self.Logger.info("Restore client sync state: We have a print cookie, detected a paused print, and timers aren't running. So there's nothing to do.") + return + + # If we are here, there's a print running or paused and we aren't tracking it currently. + # This scenario probably is due to the plugin restarting. - # Always set the file name, if not None - if fileName_CanBeNone is not None: - self._updateCurrentFileName(fileName_CanBeNone) + # This function will take the print cookie and (hopefully) recover an existing print info. + # If it can't recover an existing print info, it will create a new one. + self._RecoverOrRestForNewPrint(printCookie) # Disable the first layer complete logic, since we don't know what the base z-axis was self.HasSendFirstLayerDoneMessage = True @@ -236,19 +268,21 @@ def OnRestorePrintIfNeeded(self, isPrinting:bool, isPaused:bool, fileName_CanBeN # Make sure the timers are set correctly if isPrinting: - # If we have a total duration, use it to offset the "hours reported" so our time based notifications - # are correct. + # If we can get a duration, set the hours reported to that. hoursReportedInt = 0 - if totalDurationFloatSec_CanBeNone is not None: + durationSec = self.GetCurrentDurationSecFloat() + if durationSec > 0: # Convert seconds to hours, floor the value, make it an int. - hoursReportedInt = int(math.floor(totalDurationFloatSec_CanBeNone / 60.0 / 60.0)) + hoursReportedInt = int(math.floor(durationSec / 60.0 / 60.0)) # Setup the timers, with hours reported, to make sure that the ping timer and Gadget are running. - self.Logger.info("Restore client sync state: Restoring printing timer with existing duration of "+str(totalDurationFloatSec_CanBeNone)) + self.Logger.info("Restore client sync state: Restoring printing timer with existing duration of "+str(durationSec)) self.StartPrintTimers(False, hoursReportedInt) else: # On paused, make sure they are stopped. self.StopTimers() + self.Logger.info("Restore client sync state: Restoring into a paused print state.") + # Only used for testing. @@ -273,20 +307,43 @@ def OnGadgetPaused(self): # Fired when a print starts. - def OnStarted(self, fileName:str, fileSizeKBytes:int, totalFilamentUsageMm:int): + # The print cookie is required. It's a per platform print unique string that's used to identify the print. + # The string can be anything, but it must be a valid file name. + # The string should also be unique between prints, but common for the same print. This allows us to pull up the print info for the same print if we crash or + # or lose the printer connection. + def OnStarted(self, printCookie:str, fileName:str = None, fileSizeKBytes:int = 0, totalFilamentUsageMm:int = 0): + # Validate if self._shouldIgnoreEvent(fileName): return - self.ResetForNewPrint(None) + if printCookie is None or len(printCookie) == 0: + raise Exception("NotificationHandler OnStarted called with no print cookie.") + + # Since know we are starting a new print, we want to clear any existing print infos. + # This is important for Moonraker, because there's no way to differentiate between prints beyond the filename. + # So we have to use the file name, so we can still restore will work. + # But in the case of printing the same print back to back, the print cookie will be the same. + PrintInfoManager.Get().ClearAllPrintInfos() + + # This will reset the class for this new print and create the print info. + self._RecoverOrRestForNewPrint(printCookie) + + # Update vars self._updateCurrentFileName(fileName) - self.CurrentFileSizeInKBytes = fileSizeKBytes - self.CurrentEstFilamentUsageMm = totalFilamentUsageMm + + pi = self.GetPrintInfo() + if pi is None: + self.Logger.error("No print info returned after a new print started, this should not be possible.") + return + pi.SetFileSizeKBytes(fileSizeKBytes) + pi.SetEstFilamentUsageMm(totalFilamentUsageMm) + self.StartPrintTimers(True, None) self._sendEvent("started") - self.Logger.info(f"New print started; PrintId: {str(self.PrintId)} file:{str(self.CurrentFileName)} size:{str(self.CurrentFileSizeInKBytes)} filament:{str(self.CurrentEstFilamentUsageMm)}") + self.Logger.info(f"New print started; PrintId: {str(self.GetPrintId())} file:{str(pi.GetFileName())} size:{str(pi.GetFileSizeKBytes())} filament:{str(pi.GetEstFilamentUsageMm())}") # Fired when a print fails - def OnFailed(self, fileName, durationSecStr, reason): + def OnFailed(self, fileName:str, durationSecStr:str = None, reason:str = None): if self._shouldIgnoreEvent(fileName): return self._updateCurrentFileName(fileName) @@ -297,17 +354,17 @@ def OnFailed(self, fileName, durationSecStr, reason): # Fired when a print done # For moonraker, these vars aren't known, so they are None - def OnDone(self, fileName_CanBeNone, durationSecStr_CanBeNone): - if self._shouldIgnoreEvent(fileName_CanBeNone): + def OnDone(self, fileName:str = None, durationSecStr:str = None): + if self._shouldIgnoreEvent(fileName): return - self._updateCurrentFileName(fileName_CanBeNone) - self._updateToKnownDuration(durationSecStr_CanBeNone) + self._updateCurrentFileName(fileName) + self._updateToKnownDuration(durationSecStr) self.StopTimers() self._sendEvent("done", useFinalSnapSnapshot=True) # Fired when a print is paused - def OnPaused(self, fileName): + def OnPaused(self, fileName:str = None): if self._shouldIgnoreEvent(fileName): return @@ -331,7 +388,7 @@ def OnPaused(self, fileName): # Fired when a print is resumed - def OnResume(self, fileName): + def OnResume(self, fileName:str = None): if self._shouldIgnoreEvent(fileName): return self._updateCurrentFileName(fileName) @@ -363,7 +420,7 @@ def OnWaiting(self): if self._shouldIgnoreEvent(): return # Make this the same as the paused command. - self.OnPaused(self.CurrentFileName) + self.OnPaused() # Fired when we get a M600 command from the printer to change the filament @@ -863,12 +920,16 @@ def GetNotificationSnapshot(self, snapshotResizeParams = None): return None - # Assuming the current time is set at the start of the printer correctly + # Assuming the current time is set at the start of the printer correctly. + # This is also a live duration, if this is called once the print is over it will keep incrementing. def GetCurrentDurationSecFloat(self): - return float(time.time() - self.CurrentPrintStartTime) + pi = self.GetPrintInfo() + if pi is None: + return 0.0 + return float(time.time() - pi.GetLocalPrintStartTimeSec()) - # When OctoPrint tells us the duration, make sure we are in sync. + # If we get a known duration from the platform, be sure to update it. def _updateToKnownDuration(self, durationSecStr): # If the string is empty or None, return. # This is important for Moonraker @@ -877,17 +938,23 @@ def _updateToKnownDuration(self, durationSecStr): # If we fail this logic don't kill the event. try: - self.CurrentPrintStartTime = time.time() - float(durationSecStr) + pi = self.GetPrintInfo() + if pi is None: + return + pi.SetLocalPrintStartTimeSec(time.time() - float(durationSecStr)) except Exception as e: Sentry.ExceptionNoSend("_updateToKnownDuration exception", e) # Updates the current file name, if there is a new name to set. - def _updateCurrentFileName(self, fileNameStr): + def _updateCurrentFileName(self, fileName:str): # The None check is important for Moonraker - if fileNameStr is None or len(fileNameStr) == 0: + if fileName is None or len(fileName) == 0: return - self.CurrentFileName = fileNameStr + pi = PrintInfoManager.Get().GetPrintInfo(self.PrintCookie) + if pi is None: + return + pi.SetFileName(fileName) # Stops the final snap object if it's running and returns @@ -1042,16 +1109,22 @@ def BuildCommonEventArgs(self, event, args=None, progressOverwriteFloat=None, sn if args is None: args = {} + # Get the print info for the current print. + pi = PrintInfoManager.Get().GetPrintInfo(self.PrintCookie) + if pi is None: + self.Logger.error("NotificationsHandler failed to get the print info for the current print.") + return args + # Add the required vars args["PrinterId"] = self.PrinterId - args["PrintId"] = self.PrintId + args["PrintId"] = pi.GetPrintId() args["OctoKey"] = self.OctoKey args["Event"] = event # Always add the file name and other common props - args["FileName"] = str(self.CurrentFileName) - args["FileSizeKb"] = str(self.CurrentFileSizeInKBytes) - args["FilamentUsageMm"] = str(self.CurrentEstFilamentUsageMm) + args["FileName"] = str(pi.GetFileName()) + args["FileSizeKb"] = str(pi.GetFileSizeKBytes()) + args["FilamentUsageMm"] = str(pi.GetEstFilamentUsageMm()) # Always include the ETA, note this will be -1 if the time is unknown. timeRemainEstStr = str(self.PrinterStateInterface.GetPrintTimeRemainingEstimateInSeconds()) @@ -1121,7 +1194,7 @@ def StopFirstLayerTimer(self): # Starts all print timers, including the progress time, Gadget, and the first layer watcher. - def StartPrintTimers(self, resetHoursReported, restoreActionSetHoursReportedInt_OrNone): + def StartPrintTimers(self, resetHoursReported:bool, restoreActionSetHoursReported:int = None): # First, stop any timer that's currently running. self.StopTimers() @@ -1130,8 +1203,8 @@ def StartPrintTimers(self, resetHoursReported, restoreActionSetHoursReportedInt_ self.PingTimerHoursReported = 0 # If this is a restore, set the value - if restoreActionSetHoursReportedInt_OrNone is not None: - self.PingTimerHoursReported = int(restoreActionSetHoursReportedInt_OrNone) + if restoreActionSetHoursReported is not None: + self.PingTimerHoursReported = int(restoreActionSetHoursReported) # Setup the progress timer intervalSec = 60 * 60 # Fire every hour. @@ -1154,11 +1227,6 @@ def _IsPingTimerRunning(self): return self.ProgressTimer is not None - # Returns if we have a current print file name, indication if we are setup to track a print at all, even a paused one. - def _HasCurrentPrintFileName(self): - return self.CurrentFileName is not None and len(self.CurrentFileName) > 0 - - # Fired when the ping timer fires. def ProgressTimerCallback(self): @@ -1219,7 +1287,10 @@ def _shouldIgnoreEvent(self, fileName:str = None) -> bool: # If not, fall back to the current file name. # If there is neither, dont ignore. if fileName is None or len(fileName) == 0: - fileName = self.CurrentFileName + pi = self.GetPrintInfo() + if pi is None: + return False + fileName = pi.GetFileName() if fileName is None or len(fileName) == 0: return False # One case we want to ignore is when the continuous print plugin uses it's "placeholder" .gcode files. diff --git a/octoeverywhere/printinfo.py b/octoeverywhere/printinfo.py new file mode 100644 index 0000000..c77f17e --- /dev/null +++ b/octoeverywhere/printinfo.py @@ -0,0 +1,235 @@ +import os +import json +import time +import logging +from pathlib import Path + +# The goal of this class is to keep track of info about the current print. +# This is needed because sometimes we only get the info once, like at the start of a print, and then we want to keep it around for future notifications. +# This class also writes out to disk, so for hosts where the host can crash or be restarted mid print, the print info can be recovered. +class PrintInfo: + + # Required Json Vars + c_PrintCookieKey = "PrintCookie" + c_PrintIdKey = "PrintId" + c_PrintStartTimeSecKey = "PrintStartTimeSec" + + # Optional + c_FileNameKey = "FileName" + c_FileSizeInKBytes = "FileSizeKBytes" + c_EstFilamentUsageMm = "EstFilamentUsageMm" + c_FinalPrintDurationSec = "FinalPrintDurationSec" + + # Given a file path, this loads a print info if possible. + # Returns None on failure. + @staticmethod + def LoadFromFile(logger:logging.Logger, filePath:str): + try: + with open(filePath, "r", encoding="utf-8") as f: + data = json.load(f) + # Ensure it has the required vars. + if PrintInfo.c_PrintIdKey not in data or PrintInfo.c_PrintCookieKey not in data or PrintInfo.c_PrintStartTimeSecKey not in data: + raise Exception("File loaded, but there was no Print ID") + return PrintInfo(logger, filePath, data) + except Exception as e: + logger.error(f"Failed to load print info from file. {e}") + return None + + + # Given a file path and required args, creates a new print context. + # This will always return a PrintInfo! Even if it fails to write to disk. + @staticmethod + def CreateNew(logger:logging.Logger, filePath:str, printCookie:str, printId:str): + data = { + PrintInfo.c_PrintCookieKey : printCookie, + PrintInfo.c_PrintIdKey : printId, + PrintInfo.c_PrintStartTimeSecKey : time.time() + } + pi = PrintInfo(logger, filePath, data) + # Save, but always return a object even if this fails. + pi.Save() + return pi + + + def __init__(self, logger:logging.Logger, filePath:str, data:dict) -> None: + self.Logger = logger + self.FilePath = filePath + self.Data = data + + + # Required var, this will always exist and can't be changed. + def GetPrintId(self) -> str: + return self.Data[PrintInfo.c_PrintIdKey] + + + # Required var, this will always exist and can't be changed. + def GetPrintCookie(self) -> str: + return self.Data[PrintInfo.c_PrintCookieKey] + + + # Always exists, but it can be updated if the platform reports an exact time. + def GetLocalPrintStartTimeSec(self) -> float: + return self.Data[PrintInfo.c_PrintStartTimeSecKey] + def SetLocalPrintStartTimeSec(self, startTimeSec:float) -> float: + if self.GetLocalPrintStartTimeSec() != startTimeSec: + self.Data[PrintInfo.c_PrintStartTimeSecKey] = startTimeSec + self.Save() + + + # The file name is optional. + def GetFileName(self) -> str: + return self.Data.get(PrintInfo.c_FileNameKey, None) + def SetFileName(self, fileName:str) -> None: + current = self.GetFileName() + if current is None or current != fileName: + self.Data[PrintInfo.c_FileNameKey] = fileName + self.Save() + + + # The file size in kbytes is optional + def GetFileSizeKBytes(self) -> int: + return self.Data.get(PrintInfo.c_FileSizeInKBytes, 0) + def SetFileSizeKBytes(self, sizeBytes:int) -> None: + if self.GetFileSizeKBytes() != sizeBytes: + self.Data[PrintInfo.c_FileSizeInKBytes] = sizeBytes + self.Save() + + + # Estimated filament usage is optional. + def GetEstFilamentUsageMm(self) -> int: + return self.Data.get(PrintInfo.c_EstFilamentUsageMm, 0) + def SetEstFilamentUsageMm(self, estMm:int) -> None: + if self.GetEstFilamentUsageMm() != estMm: + self.Data[PrintInfo.c_EstFilamentUsageMm] = estMm + self.Save() + + + # This is only set when the print is done. + # Returns None if there isn't one. + def GetFinalPrintDurationSec(self) -> int: + return self.Data.get(PrintInfo.c_FinalPrintDurationSec, None) + def SetFinalPrintDurationSec(self, totalDurationSec:int) -> None: + self.Data[PrintInfo.c_FinalPrintDurationSec] = int(totalDurationSec) + self.Save() + + + # Right now this is only used by Bambu, because the printer doesn't report the + # entire print duration or when it started. So we have to calculate it ourselves. + def GetPrintDurationSec(self) -> int: + # If we have a final print duration, use it. + finalPrintDurationSec = self.GetFinalPrintDurationSec() + if finalPrintDurationSec is not None: + return int(finalPrintDurationSec) + # Otherwise, use the time since start. + return int(time.time() - self.GetLocalPrintStartTimeSec()) + + + def Save(self) -> bool: + try: + with open(self.FilePath, "w", encoding="utf-8") as f: + json.dump(self.Data, f) + return True + except Exception as e: + self.Logger.error(f"Failed to write print context from file. {e}") + return False + + +# The goal of this class is to manage the current print info. +# Ideally, the info should always be in memory, so we don't have to read it from disk. +# But if the host crashes, we can recover the print info from disk. +# This class also cleans up and old print info contexts on disk. +class PrintInfoManager: + + c_ContextsFolder = "PrintInfos" + + _Instance = None + + @staticmethod + def Init(logger:logging.Logger, localStorageFolderPath:str): + PrintInfoManager._Instance = PrintInfoManager(logger, localStorageFolderPath) + + + @staticmethod + def Get(): + return PrintInfoManager._Instance + + + def __init__(self, logger:logging.Logger, localStorageFolderPath:str) -> None: + self.Logger = logger + self.ContextFolderPath = os.path.join(localStorageFolderPath, PrintInfoManager.c_ContextsFolder) + Path(self.ContextFolderPath).mkdir(parents=True, exist_ok=True) + self.CurrentContext:PrintInfo = None + + + # Given a print cookie, if a print info. + # This print cookie should be as unique as possible, so print's dont get mixed up. + # This cleans up all contexts on disk that dont match the requested cookie. + # Returns None if no context is found for the given cookie. + def GetPrintInfo(self, printCookie:str) -> PrintInfo: + try: + # If there's no cookie, return None. + if printCookie is None: + return None + + # First, see if the current context matches. + c = self.CurrentContext + if c is not None and c.GetPrintCookie() == printCookie: + return c + + # Else, go through the files looking for the correct context. + dirAndFiles = os.listdir(self.ContextFolderPath) + printCookieFileName = self._GetPrintCookieFileName(printCookie) + context = None + # Iterate all files. Any file that doesn't match or fails to parse we delete. + for name in dirAndFiles: + fullPath = os.path.join(self.ContextFolderPath, name) + if os.path.isfile(fullPath): + if name == printCookieFileName: + context = PrintInfo.LoadFromFile(self.Logger, fullPath) + if context is None: + self._DeleteFile(fullPath) + else: + self._DeleteFile(fullPath) + else: + self._DeleteFile(fullPath) + # Always replace the current context even if it's empty, so the old context is removed. + self.CurrentContext = context + return context + except Exception as e: + self.Logger.error(f"Exception in PrintContextTracker.GetContext: {e}") + return None + + + # Clears all print infos. Note this should only be used when we absolutely know this is a new print start, + # like on a new print start or something. + def ClearAllPrintInfos(self) -> None: + try: + dirAndFiles = os.listdir(self.ContextFolderPath) + for name in dirAndFiles: + fullPath = os.path.join(self.ContextFolderPath, name) + self._DeleteFile(fullPath) + except Exception as e: + self.Logger.error(f"Exception in PrintContextTracker.ClearAllPrintInfos: {e}") + + + # Creates a new Print Info and returns it. + # This will always return a new PrintInfo, even if it fails to write to disk. + def CreateNewPrintInfo(self, printCookie:str, printId:str) -> PrintInfo: + try: + fullPath = os.path.join(self.ContextFolderPath, self._GetPrintCookieFileName(printCookie)) + self.CurrentContext = PrintInfo.CreateNew(self.Logger, fullPath, printCookie, printId) + return self.CurrentContext + except Exception as e: + self.Logger.error(f"Exception in PrintContextTracker.CreateNew: {e}") + return None + + + def _GetPrintCookieFileName(self, printCookie:str): + return f"{printCookie}.json" + + + def _DeleteFile(self, filePath:str): + try: + os.remove(filePath) + except Exception as e: + self.Logger.error(f"Exception in PrintContextTracker._DeleteFile: {e}") diff --git a/octoprint_octoeverywhere/__init__.py b/octoprint_octoeverywhere/__init__.py index 43aa6f7..9faa71f 100644 --- a/octoprint_octoeverywhere/__init__.py +++ b/octoprint_octoeverywhere/__init__.py @@ -2,6 +2,7 @@ from __future__ import absolute_import import threading import socket +import time from datetime import datetime import flask @@ -19,6 +20,7 @@ from octoeverywhere.hostcommon import HostCommon from octoeverywhere.Proto.ServerHost import ServerHost from octoeverywhere.commandhandler import CommandHandler +from octoeverywhere.printinfo import PrintInfoManager from octoeverywhere.compat import Compat @@ -175,6 +177,9 @@ def on_startup(self, host, port): # Init the mdns helper MDns.Init(self._logger, self.get_plugin_data_folder()) + # Init the print info manager. + PrintInfoManager.Init(self._logger, self.get_plugin_data_folder()) + # Setup our printer state object, that implements the interface. printerStateObject = PrinterStateObject(self._logger, self._printer) @@ -407,7 +412,11 @@ def on_event(self, event, payload): totalFilamentUsageMm = 0 if self._exists(currentData, "job") and self._exists(currentData["job"], "filament") and self._exists(currentData["job"]["filament"], "tool0") and self._exists(currentData["job"]["filament"]["tool0"], "length"): totalFilamentUsageMm = int(currentData["job"]["filament"]["tool0"]["length"]) - self.NotificationHandler.OnStarted(fileName, fileSizeKBytes, totalFilamentUsageMm) + # On OctoPrint, we dont need to support print recovery, because if this process crashes so does the print. + # So for the print cookie, we just use the current time, to make sure it's always unique. + # See details in NotificationHandler._RecoverOrRestForNewPrint + # TODO - With things like OctoKlipper, I'm not sure if the above is true, OctoPrint could restart and the print would still be active. + self.NotificationHandler.OnStarted(f"{int(time.time())}", fileName, fileSizeKBytes, totalFilamentUsageMm) elif event == "PrintFailed": fileName = self.GetDictStringOrEmpty(payload, "name") durationSec = self.GetDictStringOrEmpty(payload, "time") diff --git a/octoprint_octoeverywhere/__main__.py b/octoprint_octoeverywhere/__main__.py index 981dea5..8dc1eee 100644 --- a/octoprint_octoeverywhere/__main__.py +++ b/octoprint_octoeverywhere/__main__.py @@ -14,6 +14,7 @@ from octoeverywhere.mdns import MDns from octoeverywhere.notificationshandler import NotificationsHandler from octoeverywhere.Proto.ServerHost import ServerHost +from octoeverywhere.printinfo import PrintInfoManager from octoeverywhere.compat import Compat #from .threaddebug import ThreadDebug @@ -200,6 +201,9 @@ def GeneratePrinterId(): if LocalServerAddress is not None: OctoPingPong.Get().DisablePrimaryOverride() + # Setup the print info manager before the notification manager + PrintInfoManager.Init(logger, PluginFilePathRoot) + # Setup the notification handler. NotificationHandlerInstance = NotificationsHandler(logger, MockPrinterStateObject(logger)) diff --git a/octoprint_octoeverywhere/octoprintcommandhandler.py b/octoprint_octoeverywhere/octoprintcommandhandler.py index 1fe54fd..81aec6a 100644 --- a/octoprint_octoeverywhere/octoprintcommandhandler.py +++ b/octoprint_octoeverywhere/octoprintcommandhandler.py @@ -63,9 +63,11 @@ def GetCurrentJobStatus(self): estTotalFilamentUsageMm = int(currentData["job"]["filament"]["tool0"]["length"]) # Get the error, if there is one. + # This is shown to the user directly, so it must be short (think of a dashboard status) and formatted well. + # TODO - Since OctoPrint can give back all kinds of strings for this, we don't set it, since we can't show it to the user. errorStr_CanBeNone = None - if self._Exists(currentData, "state") and self._Exists(currentData["state"], "error"): - errorStr_CanBeNone = currentData["state"]["error"] + # if self._Exists(currentData, "state") and self._Exists(currentData["state"], "error"): + # errorStr_CanBeNone = currentData["state"]["error"] # Map the state to our common states. # We us this get_state_id to get a more explicit state, over what's in get_current_data above. diff --git a/setup.py b/setup.py index 1764a4e..549babd 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ # # This is not an installer script! -# If you're trying to install OctoEverywhere for Klipper, you want to use the ./install.sh script! +# If you're trying to install OctoEverywhere for Klipper or Bambu Connect, you want to use the ./install.sh script! # # This PY script is required for the OctoPrint plugin install process. # @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this is also parsed by the moonraker module to pull the version, so the string and format must remain the same! -plugin_version = "2.12.1" +plugin_version = "3.0.0" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 4bf0bc22f5ae6f48da1f505fbd061fbab88a59d3 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Fri, 8 Mar 2024 17:08:15 -0800 Subject: [PATCH 039/328] Fixing a few Bambu Connect bugs found by our community! --- bambu_octoeverywhere/bambustatetranslater.py | 3 ++- install.sh | 6 ++--- py_installer/Installer.py | 2 +- .../NetworkConnectors/BambuConnector.py | 27 ++++++++++--------- py_installer/Service.py | 2 +- setup.py | 2 +- 6 files changed, 23 insertions(+), 19 deletions(-) diff --git a/bambu_octoeverywhere/bambustatetranslater.py b/bambu_octoeverywhere/bambustatetranslater.py index 73408a1..b13ea3a 100644 --- a/bambu_octoeverywhere/bambustatetranslater.py +++ b/bambu_octoeverywhere/bambustatetranslater.py @@ -79,7 +79,8 @@ def OnMqttMessage(self, msg:dict, bambuState:BambuState, isFirstFullSyncResponse # and we are currently tacking a print. if not isFirstFullSyncResponse and self.NotificationsHandler.IsTrackingPrint(): # Percentage progress update - if "mc_percent" in msg["print"]: + printMsg = msg.get("print", None) + if printMsg is not None and "mc_percent" in printMsg: self.BambuOnPrintProgress(bambuState) # Since bambu doesn't tell us a print duration, we need to figure out when it ends ourselves. diff --git a/install.sh b/install.sh index a7a46e7..e5d4d42 100755 --- a/install.sh +++ b/install.sh @@ -3,7 +3,7 @@ # -# OctoEverywhere for Klipper And Bambu Labs! +# OctoEverywhere for Klipper And Bambu Lab Printers! # # Use this script to install the OctoEverywhere plugin for: # OctoEverywhere for Klipper - Where this device is running Moonraker. @@ -325,12 +325,12 @@ cat << EOF @@@@@@@@@@@@@@@,,,,,,,,,,,,,,,,,,,,,@@@@@@@@@@@@@@ EOF log_blank -log_header " OctoEverywhere For Klipper And Bambu Labs" +log_header " OctoEverywhere For Klipper And Bambu Lab Printers" log_blue " The 3D Printing Communities #1 Remote Access And AI Cloud Service" log_blank log_blank log_important "OctoEverywhere empowers the worldwide maker community with..." -log_info " - Free & Unlimited Mainsail, Fluidd, And Bambu Labs Printers Remote Access" +log_info " - Free & Unlimited Mainsail, Fluidd, And Bambu Lab Printers Remote Access" log_info " - Free & Unlimited Next-Gen AI Print Failure Detection" log_info " - Free Full Frame Rate & Full Resolution Webcam Streaming" log_info " - 5 Star Rated iOS & Android Apps" diff --git a/py_installer/Installer.py b/py_installer/Installer.py index d0bd609..30cc306 100644 --- a/py_installer/Installer.py +++ b/py_installer/Installer.py @@ -211,7 +211,7 @@ def PrintHelp(self): Logger.Info(" - OctoEverywhere for Klipper - Where Moonraker is running on this device.") Logger.Info(" - OctoEverywhere for Creality - Where this device is a Creality device (Sonic Pad, K1, Ender v3, etc)") Logger.Info(" - OctoEverywhere Companion - Where this plugin will connect to Moonraker running on a different device on the same LAN.") - Logger.Info(" - OctoEverywhere Bambu Connect - Where this plugin will connect to a Bambu Labs printer on the same LAN.") + Logger.Info(" - OctoEverywhere Bambu Connect - Where this plugin will connect to a Bambu Lab printer on the same LAN.") Logger.Blank() Logger.Warn("This installer is NOT for:") Logger.Info(" - OctoPrint or OctoKlipper - If you're using OctoPrint, install OctoEverywhere directly in OctoPrint from the plugin manager.") diff --git a/py_installer/NetworkConnectors/BambuConnector.py b/py_installer/NetworkConnectors/BambuConnector.py index 82bd40a..1be2b5e 100644 --- a/py_installer/NetworkConnectors/BambuConnector.py +++ b/py_installer/NetworkConnectors/BambuConnector.py @@ -5,7 +5,7 @@ from py_installer.Context import Context from py_installer.ConfigHelper import ConfigHelper -# A class that helps the user discover, connect, and setup the details required to connect to a remote Bambu Labs printer. +# A class that helps the user discover, connect, and setup the details required to connect to a remote Bambu Lab printer. class BambuConnector: @@ -19,21 +19,21 @@ def EnsureBambuConnection(self, context:Context): if ip is not None and port is not None and accessCode is not None and printerSn is not None: # Check if we can still connect. This can happen if the IP address changes, the user might need to setup the printer again. Logger.Info(f"Existing bambu config found. IP: {ip} - {printerSn}") - Logger.Info("Checking if we can connect to your Bambu Labs printer...") + Logger.Info("Checking if we can connect to your Bambu Lab printer...") result = NetworkSearch.ValidateConnection_Bambu(Logger.GetPyLogger(), ip, accessCode, printerSn, portStr=port, timeoutSec=10.0) if result.Success(): - Logger.Info("Successfully connected to you Bambu Labs printer!") + Logger.Info("Successfully connected to you Bambu Lab printer!") return else: # Let the user keep this connection setup, or try to set it up again. Logger.Blank() Logger.Warn(f"We failed to connect or authenticate to your printer using {ip}.") - if Util.AskYesOrNoQuestion("Do you want to setup the Bambu Labs printer connection again?") is False: - Logger.Info(f"Keeping the existing Bambu Labs printer connection setup. {ip} - {printerSn}") + if Util.AskYesOrNoQuestion("Do you want to setup the Bambu Lab printer connection again?") is False: + Logger.Info(f"Keeping the existing Bambu Lab printer connection setup. {ip} - {printerSn}") return ipOrHostname, port, accessToken, printerSn = self._SetupNewBambuConnection() - Logger.Info(f"You Bambu printer was found and authentication was successful! IP: {ip}") + Logger.Info(f"You Bambu printer was found and authentication was successful! IP: {ipOrHostname}") ConfigHelper.WriteCompanionDetails(context, ipOrHostname, port) ConfigHelper.WriteBambuDetails(context, accessToken, printerSn) @@ -50,10 +50,10 @@ def _SetupNewBambuConnection(self): Logger.Blank() Logger.Blank() Logger.Header("##################################") - Logger.Header(" Bambu Labs Printer Setup") + Logger.Header(" Bambu Lab Printer Setup") Logger.Header("##################################") Logger.Blank() - Logger.Info("OctoEverywhere Bambu Connect needs to connect to your Bambu Labs printer to provide remote access.") + Logger.Info("OctoEverywhere Bambu Connect needs to connect to your Bambu Lab printer to provide remote access.") Logger.Info("Bambu Connect needs your printer's Access Code and Serial Number to connect to your printer.") Logger.Info("If you have any trouble, we are happy to help! Contact us at support@octoeverywhere.com") @@ -62,15 +62,18 @@ def _SetupNewBambuConnection(self): while True: Logger.Blank() Logger.Blank() - Logger.Header("We need your Bambu Labs printer's Access Code to connect.") + Logger.Header("We need your Bambu Lab printer's Access Code to connect.") Logger.Info("The Access Code can be found using the screen on your printer, in the Network settings.") Logger.Blank() Logger.Warn("Follow this link for a step-by-step guide to find the Access Code for your printer:") Logger.Warn("https://octoeverywhere.com/s/access-code") Logger.Blank() + Logger.Info("The access code is case sensitive - make sure to enter it exactly as shown on your printer.") + Logger.Blank() accessCode = input("Enter your printer's Access Code: ") # Validate + # The access code IS CASE SENSITIVE and can letters and numbers. accessCode = accessCode.strip() if len(accessCode) != 8: if Util.AskYesOrNoQuestion(f"The Access Code should be 8 numbers, you have entered {len(accessCode)}. Do you want to try again? "): @@ -78,8 +81,8 @@ def _SetupNewBambuConnection(self): retryEntry = False for c in accessCode: - if not c.isdigit(): - if Util.AskYesOrNoQuestion("The Access Code should only be numbers, you seem to have entered something else. Do you want to try again? "): + if not c.isdigit() and not c.isalpha(): + if Util.AskYesOrNoQuestion("The Access Code should only be letters and numbers, you seem to have entered something else. Do you want to try again? "): retryEntry = True break if retryEntry: @@ -97,7 +100,7 @@ def _SetupNewBambuConnection(self): printerSn = None while True: Logger.Blank() - Logger.Header("Finally, Bambu Connect needs your Bambu Labs printer's Serial Number to connect.") + Logger.Header("Finally, Bambu Connect needs your Bambu Lab printer's Serial Number to connect.") Logger.Info("The Serial Number is required for authentication when the printer's local network protocol.") Logger.Info("Your Serial Number and Access Code are only stored on this device and will not be uploaded.") Logger.Blank() diff --git a/py_installer/Service.py b/py_installer/Service.py index f85552b..d032009 100644 --- a/py_installer/Service.py +++ b/py_installer/Service.py @@ -60,7 +60,7 @@ def Install(self, context:Context): # Install for debian setups def _InstallDebian(self, context:Context, argsJsonBase64:str, moduleNameToRun:str): - serviceName = "Bambu Labs Printers" if context.IsBambuSetup else "Moonraker" + serviceName = "Bambu Lab Printers" if context.IsBambuSetup else "Moonraker" optionalAfter = "" if context.IsBambuSetup else "moonraker.service" s = f'''\ # OctoEverywhere For {serviceName} diff --git a/setup.py b/setup.py index 549babd..efc1261 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this is also parsed by the moonraker module to pull the version, so the string and format must remain the same! -plugin_version = "3.0.0" +plugin_version = "3.0.1" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 8c57e78c7cebc1158ea8321f3ee46911d28412b4 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sat, 9 Mar 2024 10:22:14 -0800 Subject: [PATCH 040/328] Disabling the OctoPrint check for companion or bambu installs. --- install.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/install.sh b/install.sh index e5d4d42..fb3c255 100755 --- a/install.sh +++ b/install.sh @@ -281,6 +281,13 @@ check_for_octoprint() # Skip, there's no need and we don't have curl. return else + # Check if we are running in the Bambu Connect or Companion mode, if so, don't do this since + # The device could be running OctoPrint and that's fine. + if [[ "$*" == *"-bambu"* ]] || [[ "$*" == *"-companion"* ]] + then + return + fi + # Do a basic check to see if OctoPrint is running on the standard port. # This obviously doesn't work for all OctoPrint setups, but it works for the default ones. if curl -s "http://127.0.0.1:5000" >/dev/null ; then @@ -359,7 +366,7 @@ install_or_update_system_dependencies # Check that OctoPrint isn't found. If it is, we want to check with the user to make sure they are # not trying to setup OE for OctoPrint. -check_for_octoprint +check_for_octoprint $* # Now make sure the virtual env exists, is updated, and all of our currently required PY packages are updated. install_or_update_python_env From ca680deb6e1f75387b1f4c87220c922b10011b65 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sat, 9 Mar 2024 16:03:37 -0800 Subject: [PATCH 041/328] Fixing a bug where the printer reports an error, but isn't in an error state. --- bambu_octoeverywhere/bambumodels.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bambu_octoeverywhere/bambumodels.py b/bambu_octoeverywhere/bambumodels.py index 0930fa5..5dd25c4 100644 --- a/bambu_octoeverywhere/bambumodels.py +++ b/bambu_octoeverywhere/bambumodels.py @@ -115,6 +115,14 @@ def GetPrinterError(self) -> BambuPrintErrors: # If there is a printer error, this is not 0 if self.print_error is None or self.print_error == 0: return None + + # Oddly there are some errors that aren't errors? And the printer might sit in them while printing. + # We ignore these. We also use the direct int values, so we don't have to build the hex string all of the time. + # These error codes are in https://e.bambulab.com/query.php?lang=en, but have empty strings. + # Hex: 05008030, 03008012, 0500C011 + if self.print_error == 83918896 or self.print_error == 50364434 or self.print_error == 83935249: + return None + # There's a full list of errors here, we only care about some of them # https://e.bambulab.com/query.php?lang=en # We format the error into a hex the same way the are on the page, to make it easier. From 5209edb451d5a87f7a20ca5d93e78842d4a4e08d Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sat, 9 Mar 2024 16:18:10 -0800 Subject: [PATCH 042/328] Minor text change. --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index fb3c255..76e0203 100755 --- a/install.sh +++ b/install.sh @@ -332,7 +332,7 @@ cat << EOF @@@@@@@@@@@@@@@,,,,,,,,,,,,,,,,,,,,,@@@@@@@@@@@@@@ EOF log_blank -log_header " OctoEverywhere For Klipper And Bambu Lab Printers" +log_header " OctoEverywhere For Klipper, Creality OS, And Bambu Lab Printers" log_blue " The 3D Printing Communities #1 Remote Access And AI Cloud Service" log_blank log_blank From 0f521801f24c08a4c5a3dfbc90ca279c85f5dcd1 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sat, 9 Mar 2024 16:20:14 -0800 Subject: [PATCH 043/328] Version bump --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index efc1261..d2d57aa 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this is also parsed by the moonraker module to pull the version, so the string and format must remain the same! -plugin_version = "3.0.1" +plugin_version = "3.0.2" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 2ddaaf56f6280f23fca6c508207cc17c9f83805b Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Mon, 11 Mar 2024 21:50:55 -0700 Subject: [PATCH 044/328] Re-enabling sentry for pref and error reporting. --- octoeverywhere/sentry.py | 59 ++++++++++++++++++++-------------------- requirements.txt | 1 + setup.py | 2 ++ 3 files changed, 33 insertions(+), 29 deletions(-) diff --git a/octoeverywhere/sentry.py b/octoeverywhere/sentry.py index 38e14af..10046aa 100644 --- a/octoeverywhere/sentry.py +++ b/octoeverywhere/sentry.py @@ -1,22 +1,24 @@ -#import logging +import logging import time import traceback -# import sentry_sdk -# from sentry_sdk.integrations.logging import LoggingIntegration -# from sentry_sdk.integrations.threading import ThreadingIntegration -# from sentry_sdk import capture_exception +import sentry_sdk +from sentry_sdk.integrations.logging import LoggingIntegration +from sentry_sdk.integrations.threading import ThreadingIntegration +from sentry_sdk import capture_exception # A helper class to handle Sentry logic. class Sentry: + logger = None isDevMode = False lastErrorReport = time.time() lastErrorCount = 0 -# Sets up Sentry + + # Sets up Sentry @staticmethod - def Init(logger, versionString, isDevMode): + def Init(logger:logging.Logger, versionString:str, isDevMode:bool): # Capture the logger for future use. Sentry.logger = logger @@ -26,25 +28,24 @@ def Init(logger, versionString, isDevMode): # Only setup sentry if we aren't in dev mode. if Sentry.isDevMode is False: try: - # Disabled for now - # # We don't want sentry to capture error logs, which is it's default. # We do want the logging for breadcrumbs, so we will leave it enabled. - # sentry_logging = LoggingIntegration( - # level=logging.INFO, # Capture info and above as breadcrumbs - # event_level=logging.FATAL # Only send FATAL errors and above. - # ) - # # Setup and init - # sentry_sdk.init( - # dsn="https://a2eaa1b58ea447f08472545eedfc74fb@o1317704.ingest.sentry.io/6570908", - # integrations=[ - # sentry_logging, - # ThreadingIntegration(propagate_hub=True), - # ], - # release=versionString, - # before_send=Sentry._beforeSendFilter - # ) - pass + sentry_logging = LoggingIntegration( + level=logging.INFO, # Capture info and above as breadcrumbs + event_level=logging.FATAL # Only send FATAL errors and above. + ) + # Setup and init + sentry_sdk.init( + dsn="https://5ce4e93a61f09e32634ab4ffc7a865c0@oe-sentry.octoeverywhere.com/6", + integrations=[ + sentry_logging, + ThreadingIntegration(propagate_hub=True), + ], + release=versionString, + before_send=Sentry._beforeSendFilter, + traces_sample_rate=0.5, + profiles_sample_rate=0.1 + ) except Exception as e: logger.error("Failed to init Sentry: "+str(e)) @@ -77,10 +78,10 @@ def _beforeSendFilter(event, hint): try: stack = traceback.extract_stack((exc_info[2]).tb_frame) for s in stack: - # Check for any "octoeverywhere". The main source should be our package folder, which is - # "octoprint_octoeverywhere". + # Check for any "octoeverywhere" or "linux_host" in the filename. + # This will match one of the main modules in our code, but exclude any 3rd party code. filenameLower = s.filename.lower() - if "octoeverywhere" in filenameLower: + if "octoeverywhere" in filenameLower or "linux_host" in filenameLower or "py_installer" in filenameLower: # If found, return the event so it's reported. return event except Exception as e: @@ -118,5 +119,5 @@ def _handleException(msg, exception, sendException): # Sentry is disabled for now. # Never send in dev mode, as Sentry will not be setup. - # if sendException and Sentry.isDevMode is False: - # capture_exception(exception) + if sendException and Sentry.isDevMode is False: + capture_exception(exception) diff --git a/requirements.txt b/requirements.txt index 408a3fb..a2362ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,6 +16,7 @@ rsa>=4.9 dnspython>=2.3.0 httpx>=0.24.1,<0.26.0 urllib3>=1.26.15,<2.0.0 +sentry-sdk>=1.19.1,<2 # The following are required only for Moonraker configparser # Only used for Bambu Connect diff --git a/setup.py b/setup.py index d2d57aa..e5db52d 100644 --- a/setup.py +++ b/setup.py @@ -70,6 +70,7 @@ # certifi - We use to keep certs on the device that we need for let's encrypt. So we want to keep it fresh. # rsa - OctoPrint 1.5.3 requires RAS>=4.0, so we must leave it at 4.0. # httpx - Is an asyncio http lib. It seems to be required by dnspython, but dnspython doesn't enforce it. We had a user having an issue that updated to 0.24.0, and it resolved the issue. +# sentry-sdk - We use the same version as OctoPrint, so we don't have to worry about mismatched versions. # # Note! These also need to stay in sync with requirements.txt, for the most part they should be the exact same! plugin_requires = [ @@ -82,6 +83,7 @@ "dnspython>=2.3.0", "httpx>=0.24.1,<0.26.0", "urllib3>=1.26.18,<2.0.0" + "sentry-sdk>=1.19.1,<2" ] ### -------------------------------------------------------------------------------------------------------------------- From 5f940c947de46fb79be34a44d2053cfa71de3247 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Mon, 11 Mar 2024 21:56:06 -0700 Subject: [PATCH 045/328] Adding more plugin context to Sentry --- bambu_octoeverywhere/bambuhost.py | 5 ++++- moonraker_octoeverywhere/moonrakerhost.py | 5 ++++- octoeverywhere/sentry.py | 3 ++- octoprint_octoeverywhere/__init__.py | 2 +- octoprint_octoeverywhere/__main__.py | 2 +- setup.py | 2 +- 6 files changed, 13 insertions(+), 6 deletions(-) diff --git a/bambu_octoeverywhere/bambuhost.py b/bambu_octoeverywhere/bambuhost.py index c5e3636..02aabd5 100644 --- a/bambu_octoeverywhere/bambuhost.py +++ b/bambu_octoeverywhere/bambuhost.py @@ -49,7 +49,7 @@ def __init__(self, configDir:str, logDir:str, devConfig_CanBeNone) -> None: self.Config.SetLogger(self.Logger) # Init sentry, since it's needed for Exceptions. - Sentry.Init(self.Logger, "bambu", True) + Sentry.Init(self.Logger, "0.0.0", "bambu", True) except Exception as e: tb = traceback.format_exc() @@ -69,6 +69,9 @@ def RunBlocking(self, configPath, localStorageDir, repoRoot, devConfig_CanBeNone pluginVersionStr = Version.GetPluginVersion(repoRoot) self.Logger.info("Plugin Version: %s", pluginVersionStr) + # Re-init Sentry now that we have the plugin version. + Sentry.Init(self.Logger, pluginVersionStr, "bambu", devConfig_CanBeNone is not None) + # Before the first time setup, we must also init the Secrets class and do the migration for the printer id and private key, if needed. self.Secrets = Secrets(self.Logger, localStorageDir) diff --git a/moonraker_octoeverywhere/moonrakerhost.py b/moonraker_octoeverywhere/moonrakerhost.py index 93e8e20..d21f2b5 100644 --- a/moonraker_octoeverywhere/moonrakerhost.py +++ b/moonraker_octoeverywhere/moonrakerhost.py @@ -57,7 +57,7 @@ def __init__(self, klipperConfigDir, klipperLogDir, devConfig_CanBeNone) -> None self.Config.SetLogger(self.Logger) # Init sentry, since it's needed for Exceptions. - Sentry.Init(self.Logger, "klipper", True) + Sentry.Init(self.Logger, "0.0.0", "klipper", True) except Exception as e: tb = traceback.format_exc() @@ -82,6 +82,9 @@ def RunBlocking(self, klipperConfigDir, isCompanionMode, localStorageDir, servic pluginVersionStr = Version.GetPluginVersion(repoRoot) self.Logger.info("Plugin Version: %s", pluginVersionStr) + # Re-init Sentry now that we have the plugin version. + Sentry.Init(self.Logger, pluginVersionStr, "klipper", devConfig_CanBeNone is not None) + # This logic only works if running locally. if not isCompanionMode: # Before we do this first time setup, make sure our config files are in place. This is important diff --git a/octoeverywhere/sentry.py b/octoeverywhere/sentry.py index 10046aa..eb18532 100644 --- a/octoeverywhere/sentry.py +++ b/octoeverywhere/sentry.py @@ -18,7 +18,7 @@ class Sentry: # Sets up Sentry @staticmethod - def Init(logger:logging.Logger, versionString:str, isDevMode:bool): + def Init(logger:logging.Logger, versionString:str, distType:str, isDevMode:bool): # Capture the logger for future use. Sentry.logger = logger @@ -42,6 +42,7 @@ def Init(logger:logging.Logger, versionString:str, isDevMode:bool): ThreadingIntegration(propagate_hub=True), ], release=versionString, + dist=distType, before_send=Sentry._beforeSendFilter, traces_sample_rate=0.5, profiles_sample_rate=0.1 diff --git a/octoprint_octoeverywhere/__init__.py b/octoprint_octoeverywhere/__init__.py index 9faa71f..9e4be01 100644 --- a/octoprint_octoeverywhere/__init__.py +++ b/octoprint_octoeverywhere/__init__.py @@ -148,7 +148,7 @@ def on_startup(self, host, port): self._logger.info("OctoPrint host:" +str(self.OctoPrintLocalHost) + " port:" + str(self.OctoPrintLocalPort)) # Setup Sentry to capture issues. - Sentry.Init(self._logger, self._plugin_version, False) + Sentry.Init(self._logger, self._plugin_version, "octoprint", False) # Setup our telemetry class. Telemetry.Init(self._logger) diff --git a/octoprint_octoeverywhere/__main__.py b/octoprint_octoeverywhere/__main__.py index 8dc1eee..344264c 100644 --- a/octoprint_octoeverywhere/__main__.py +++ b/octoprint_octoeverywhere/__main__.py @@ -163,7 +163,7 @@ def GeneratePrinterId(): Compat.SetIsOctoPrint(True) # Init Sentry, but it won't report since we are in dev mode. - Sentry.Init(logger, "dev", True) + Sentry.Init(logger, "0.0.0", "dev", True) Telemetry.Init(logger) if LocalServerAddress is not None: Telemetry.SetServerProtocolAndDomain("http://"+LocalServerAddress) diff --git a/setup.py b/setup.py index e5db52d..60d8f31 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this is also parsed by the moonraker module to pull the version, so the string and format must remain the same! -plugin_version = "3.0.2" +plugin_version = "3.0.3" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 4d0841d69c9f9dad45dab7024252018bc96a8c76 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Tue, 12 Mar 2024 16:33:41 -0700 Subject: [PATCH 046/328] Fixing up a few bugs found from Sentry. --- bambu_octoeverywhere/bambuclient.py | 18 ++- bambu_octoeverywhere/bambuhost.py | 15 +- install.sh | 8 +- moonraker_octoeverywhere/moonrakerclient.py | 25 ++-- .../moonrakercredentailmanager.py | 3 + moonraker_octoeverywhere/moonrakerhost.py | 18 ++- .../WebStream/octowebstreamhttphelper.py | 17 +-- octoeverywhere/exceptions.py | 28 ++++ octoeverywhere/notificationshandler.py | 6 +- octoeverywhere/octohttprequest.py | 5 +- octoeverywhere/sentry.py | 135 +++++++++++------- octoeverywhere/webcamhelper.py | 7 + octoeverywhere/websocketimpl.py | 18 +++ octoprint_octoeverywhere/__init__.py | 4 +- octoprint_octoeverywhere/__main__.py | 3 +- setup.py | 4 +- 16 files changed, 220 insertions(+), 94 deletions(-) create mode 100644 octoeverywhere/exceptions.py diff --git a/bambu_octoeverywhere/bambuclient.py b/bambu_octoeverywhere/bambuclient.py index 53b5399..2aa65ee 100644 --- a/bambu_octoeverywhere/bambuclient.py +++ b/bambu_octoeverywhere/bambuclient.py @@ -108,6 +108,7 @@ def _ClientWorker(self): self.Client.on_message = self._OnMessage self.Client.on_disconnect = self._OnDisconnect self.Client.on_subscribe = self._OnSubscribe + self.Client.on_log = self._OnLog # Get the IP to try on this connect ipOrHostname = self._GetIpOrHostnameToTry() @@ -191,6 +192,17 @@ def _OnDisconnect(self, client, userdata, disconnect_flags, reason_code, propert self._CleanupStateOnDisconnect() + # Fired when the MQTT connection has something to log. + def _OnLog(self, client, userdata, level:mqtt.LOGGING_LEVEL, msg:str): + if level == mqtt.MQTT_LOG_ERR: + # If the string is something like "Caught exception in on_connect: ..." + # It's a leaked exception from us. + if "exception" in msg: + Sentry.Exception("MQTT leaked exception.", Exception(msg)) + else: + self.Logger.error(f"MQTT log error: {msg}") + + # Fried when the MQTT subscribe result has come back. def _OnSubscribe(self, client, userdata, mid, reason_code_list:List[mqtt.ReasonCode], properties): # We only want to listen for the result of the report subscribe. @@ -211,10 +223,10 @@ def _OnSubscribe(self, client, userdata, mid, reason_code_list:List[mqtt.ReasonC # Fired when there's an incoming MQTT message. - def _OnMessage(self, client, userdata, msg:mqtt.MQTTMessage): + def _OnMessage(self, client, userdata, mqttMsg:mqtt.MQTTMessage): try: # Try to deserialize the message. - msg = json.loads(msg.payload) + msg = json.loads(mqttMsg.payload) if msg is None: raise Exception("Parsed json MQTT message returned None") @@ -271,7 +283,7 @@ def _OnMessage(self, client, userdata, msg:mqtt.MQTTMessage): Sentry.Exception("Exception calling StateTranslator.OnMqttMessage", e) except Exception as e: - self.Logger.warn(f"Failed to handle incoming mqtt message. {e} {msg.payload}") + Sentry.Exception(f"Failed to handle incoming mqtt message. {mqttMsg.payload}", e) # Publishes a message and blocks until it knows if the message send was successful or not. diff --git a/bambu_octoeverywhere/bambuhost.py b/bambu_octoeverywhere/bambuhost.py index 02aabd5..2bb8efa 100644 --- a/bambu_octoeverywhere/bambuhost.py +++ b/bambu_octoeverywhere/bambuhost.py @@ -48,8 +48,8 @@ def __init__(self, configDir:str, logDir:str, devConfig_CanBeNone) -> None: self.Logger = LoggerInit.GetLogger(self.Config, logDir, logLevelOverride_CanBeNone) self.Config.SetLogger(self.Logger) - # Init sentry, since it's needed for Exceptions. - Sentry.Init(self.Logger, "0.0.0", "bambu", True) + # Give the logger to Sentry ASAP. + Sentry.SetLogger(self.Logger) except Exception as e: tb = traceback.format_exc() @@ -61,16 +61,17 @@ def __init__(self, configDir:str, logDir:str, devConfig_CanBeNone) -> None: def RunBlocking(self, configPath, localStorageDir, repoRoot, devConfig_CanBeNone): # Do all of this in a try catch, so we can log any issues before exiting try: - self.Logger.info("##################################") - self.Logger.info("#### OctoEverywhere Starting #####") - self.Logger.info("##################################") + self.Logger.info("################################################") + self.Logger.info("#### OctoEverywhere Bambu Connect Starting #####") + self.Logger.info("################################################") # Find the version of the plugin, this is required and it will throw if it fails. pluginVersionStr = Version.GetPluginVersion(repoRoot) self.Logger.info("Plugin Version: %s", pluginVersionStr) - # Re-init Sentry now that we have the plugin version. - Sentry.Init(self.Logger, pluginVersionStr, "bambu", devConfig_CanBeNone is not None) + # As soon as we have the plugin version, setup Sentry + # Enabling profiling and no filtering, since we are the only PY in this process. + Sentry.Setup(pluginVersionStr, "bambu", devConfig_CanBeNone is not None, enableProfiling=True, filterExceptionsByPackage=False) # Before the first time setup, we must also init the Secrets class and do the migration for the printer id and private key, if needed. self.Secrets = Secrets(self.Logger, localStorageDir) diff --git a/install.sh b/install.sh index 76e0203..8681e38 100755 --- a/install.sh +++ b/install.sh @@ -219,12 +219,14 @@ install_or_update_system_dependencies() # the user might install opkg via the 3rd party moonraker installer script. # But in general, PY will already be installed, so there's no need to try. # On the K1, the only we thing we ensure is that virtualenv is installed via pip. - pip3 install virtualenv + # We have had users report issues where this install gets stuck, using the no cache dir flag seems to fix it. + pip3 install -q --no-cache-dir virtualenv elif [[ $IS_SONIC_PAD_OS -eq 1 ]] then # The sonic pad always has opkg installed, so we can make sure these packages are installed. + # We have had users report issues where this install gets stuck, using the no cache dir flag seems to fix it. opkg install ${SONIC_PAD_DEP_LIST} - pip3 install virtualenv + pip3 install -q --no-cache-dir virtualenv else # It seems a lot of printer control systems don't have the date and time set correctly, and then the fail # getting packages and other downstream things. We will will use our HTTP API to set the current UTC time. @@ -267,7 +269,7 @@ install_or_update_python_env() # Finally, ensure our plugin requirements are installed and updated. log_info "Installing or updating required python libs..." - "${OE_ENV}"/bin/pip3 install -q -r "${OE_REPO_DIR}"/requirements.txt + "${OE_ENV}"/bin/pip3 install -q -r --require-virtualenv --no-cache-dir "${OE_REPO_DIR}"/requirements.txt log_info "Python libs installed." } diff --git a/moonraker_octoeverywhere/moonrakerclient.py b/moonraker_octoeverywhere/moonrakerclient.py index cced740..397b4fa 100644 --- a/moonraker_octoeverywhere/moonrakerclient.py +++ b/moonraker_octoeverywhere/moonrakerclient.py @@ -11,6 +11,7 @@ from octoeverywhere.sentry import Sentry from octoeverywhere.websocketimpl import Client from octoeverywhere.notificationshandler import NotificationsHandler +from octoeverywhere.exceptions import NoSentryReportException from linux_host.config import Config @@ -147,7 +148,7 @@ def _UpdateMoonrakerHostAndPort(self) -> None: if Compat.IsCompanionMode() is False: if os.path.exists(self.MoonrakerConfigFilePath) is False: self.Logger.error("Moonraker client failed to find a moonraker config. Re-run the ./install.sh script from the OctoEverywhere repo to update the path.") - raise Exception("No config file found") + raise NoSentryReportException("No moonraker config file found") # Get the values. (hostStr, portInt) = self.GetMoonrakerHostAndPortFromConfig() @@ -195,10 +196,14 @@ def GetMoonrakerHostAndPortFromConfig(self): # Done! return (currentHostStr, currentPortInt) - + except configparser.ParsingError as e: + if "Source contains parsing errors" in str(e): + self.Logger.error("Failed to parse moonraker config file. "+str(e)) + else: + Sentry.Exception("Failed to read moonraker port and host from config, assuming defaults. Host:"+currentHostStr+" Port:"+str(currentPortInt), e) except Exception as e: Sentry.Exception("Failed to read moonraker port and host from config, assuming defaults. Host:"+currentHostStr+" Port:"+str(currentPortInt), e) - return (currentHostStr, currentPortInt) + return (currentHostStr, currentPortInt) # @@ -550,9 +555,9 @@ def _AfterOpenReadyWaiter(self, targetWsObjRef): # Handle the timeout without throwing, since this happens sometimes when the system is down. if result.ErrorCode == JsonRpcResponse.OE_ERROR_TIMEOUT: - self.Logger.info("Moonraker client failed to send klippy ready query message, it hit a timeout.") - self._RestartWebsocket() - return + raise NoSentryReportException("Moonraker client failed to send klippy ready query message, it hit a timeout.") + if result.ErrorCode == JsonRpcResponse.OE_ERROR_WS_NOT_CONNECTED: + raise NoSentryReportException("Moonraker client failed to send klippy ready query message, there was no websocket.") self.Logger.error("Moonraker client failed to send klippy ready query message. "+result.GetLoggingErrorStr()) raise Exception("Error returned from klippy state query. "+ str(result.GetLoggingErrorStr())) @@ -574,7 +579,7 @@ def _AfterOpenReadyWaiter(self, targetWsObjRef): # Done return - if state == "startup" or state == "error" or state == "shutdown" or state == "initializing": + if state == "startup" or state == "error" or state == "shutdown" or state == "initializing" or state == "disconnected": logCounter += 1 # 2 seconds * 150 = one log every 5 minutes. We don't want to log a ton if the printer is offline for a long time. if logCounter % 150 == 1: @@ -689,7 +694,11 @@ def _onWsClose(self, ws): # Called if the websocket hits an error and is closing. def _onWsError(self, ws, exception): - Sentry.Exception("Exception rased from moonraker client websocket connection. The connection will be closed.", exception) + if Client.IsCommonConnectionException(exception): + # Don't bother logging, this just means there's no server to connect to. + pass + else: + Sentry.Exception("Exception rased from moonraker client websocket connection. The connection will be closed.", exception) # A helper class used for waiting rpc requests diff --git a/moonraker_octoeverywhere/moonrakercredentailmanager.py b/moonraker_octoeverywhere/moonrakercredentailmanager.py index 7d17228..7cd40f2 100644 --- a/moonraker_octoeverywhere/moonrakercredentailmanager.py +++ b/moonraker_octoeverywhere/moonrakercredentailmanager.py @@ -145,6 +145,9 @@ def _TryToFindUnixSocket(self) -> str or None: if os.path.exists(possibleMoonrakerSocketFilePath): self.Logger.info("Moonraker socket path found from moonraker config klippy socket path. :"+possibleMoonrakerSocketFilePath) return possibleMoonrakerSocketFilePath + except configparser.ParsingError as e: + if "Source contains parsing errors" in str(e): + self.Logger.error("_TryToFindUnixSocket failed to handle moonraker config. "+str(e)) except Exception as e: Sentry.Exception("_TryToFindUnixSocket failed to handle moonraker config.", e) diff --git a/moonraker_octoeverywhere/moonrakerhost.py b/moonraker_octoeverywhere/moonrakerhost.py index d21f2b5..5b80f9b 100644 --- a/moonraker_octoeverywhere/moonrakerhost.py +++ b/moonraker_octoeverywhere/moonrakerhost.py @@ -56,8 +56,8 @@ def __init__(self, klipperConfigDir, klipperLogDir, devConfig_CanBeNone) -> None self.Logger = LoggerInit.GetLogger(self.Config, klipperLogDir, logLevelOverride_CanBeNone) self.Config.SetLogger(self.Logger) - # Init sentry, since it's needed for Exceptions. - Sentry.Init(self.Logger, "0.0.0", "klipper", True) + # Set the logger ASAP. + Sentry.SetLogger(self.Logger) except Exception as e: tb = traceback.format_exc() @@ -71,9 +71,12 @@ def RunBlocking(self, klipperConfigDir, isCompanionMode, localStorageDir, servic devConfig_CanBeNone): # Do all of this in a try catch, so we can log any issues before exiting try: - self.Logger.info("##################################") - self.Logger.info("#### OctoEverywhere Starting #####") - self.Logger.info("##################################") + self.Logger.info("################################################") + if isCompanionMode: + self.Logger.info("## OctoEverywhere Klipper Companion Starting ##") + else: + self.Logger.info("##### OctoEverywhere For Klipper Starting ######") + self.Logger.info("################################################") # Set companion mode flag as soon as we know it. Compat.SetIsCompanionMode(isCompanionMode) @@ -82,8 +85,9 @@ def RunBlocking(self, klipperConfigDir, isCompanionMode, localStorageDir, servic pluginVersionStr = Version.GetPluginVersion(repoRoot) self.Logger.info("Plugin Version: %s", pluginVersionStr) - # Re-init Sentry now that we have the plugin version. - Sentry.Init(self.Logger, pluginVersionStr, "klipper", devConfig_CanBeNone is not None) + # As soon as we have the plugin version, setup Sentry + # Enabling profiling and no filtering, since we are the only PY in this process. + Sentry.Setup(pluginVersionStr, "klipper", devConfig_CanBeNone is not None, enableProfiling=True, filterExceptionsByPackage=False) # This logic only works if running locally. if not isCompanionMode: diff --git a/octoeverywhere/WebStream/octowebstreamhttphelper.py b/octoeverywhere/WebStream/octowebstreamhttphelper.py index 2a75aed..ea3682d 100644 --- a/octoeverywhere/WebStream/octowebstreamhttphelper.py +++ b/octoeverywhere/WebStream/octowebstreamhttphelper.py @@ -412,16 +412,17 @@ def executeHttpRequest(self): WebStreamMsg.AddMultipartReadsPerSecond(builder, self.MultipartReadsPerSecond) self.MultipartReadsPerSecond = 0 # Also attach the other stats. - if self.BodyReadTimeHighWaterMarkSec > 65535.0 or self.BodyReadTimeHighWaterMarkSec < 0.0: - self.Logger.warn("self.BodyReadTimeHighWaterMarkSec is larger than uint8. "+str(self.BodyReadTimeHighWaterMarkSec)) - self.BodyReadTimeHighWaterMarkSec = 65535.0 - WebStreamMsg.AddBodyReadTimeHighWaterMarkMs(builder, int(self.BodyReadTimeHighWaterMarkSec * 1000.0)) + bodyReadTimeHighWaterMarkMs = int(self.BodyReadTimeHighWaterMarkSec * 1000.0) self.BodyReadTimeHighWaterMarkSec = 0.0 - if self.ServiceUploadTimeHighWaterMarkSec > 65535.0 or self.ServiceUploadTimeHighWaterMarkSec < 0.0: - self.Logger.warn("self.ServiceUploadTimeHighWaterMarkSec is larger than uint8. "+str(self.ServiceUploadTimeHighWaterMarkSec)) - self.ServiceUploadTimeHighWaterMarkSec = 65535.0 - WebStreamMsg.AddBodyReadTimeHighWaterMarkMs(builder, int(self.ServiceUploadTimeHighWaterMarkSec * 1000.0)) + if bodyReadTimeHighWaterMarkMs > 65535 or bodyReadTimeHighWaterMarkMs < 0: + bodyReadTimeHighWaterMarkMs = 65535 + WebStreamMsg.AddBodyReadTimeHighWaterMarkMs(builder, bodyReadTimeHighWaterMarkMs) + + serviceUploadTimeHighWaterMarkMs = int(self.ServiceUploadTimeHighWaterMarkSec * 1000.0) self.ServiceUploadTimeHighWaterMarkSec = 0.0 + if serviceUploadTimeHighWaterMarkMs > 65535 or serviceUploadTimeHighWaterMarkMs < 0: + serviceUploadTimeHighWaterMarkMs = 65535 + WebStreamMsg.AddSocketSendTimeHighWaterMarkMs(builder, serviceUploadTimeHighWaterMarkMs) webStreamMsgOffset = WebStreamMsg.End(builder) diff --git a/octoeverywhere/exceptions.py b/octoeverywhere/exceptions.py new file mode 100644 index 0000000..8c72c2a --- /dev/null +++ b/octoeverywhere/exceptions.py @@ -0,0 +1,28 @@ +# This is an exception type that can be used to indicate that something bad happened, +# but we don't want to report it to Sentry because there's no logic error. +# +# For example, if we know the moonraker connection details are correct but there's no device to connect to, +# then the server is probably down and we can't do anything about that. +class NoSentryReportException(Exception): + + def __init__(self, message:str = None, exception:Exception = None): + self.Message = message + self.Exception = exception + super().__init__(message) + + + def __str__(self) -> str: + return self._GetMessage() + + + def __repr__(self) -> str: + return self._GetMessage() + + + def _GetMessage(self) -> str: + result = "No Sentry Exception - " + if self.Message is not None: + result += f"Message: `{self.Message}`" + if self.Exception is not None: + result += f" Exception: {self.Exception}" + return result diff --git a/octoeverywhere/notificationshandler.py b/octoeverywhere/notificationshandler.py index 28febf0..490e0dc 100644 --- a/octoeverywhere/notificationshandler.py +++ b/octoeverywhere/notificationshandler.py @@ -1098,7 +1098,7 @@ def _sendEventThreadWorker(self, event, args = None, progressOverwriteFloat = No # Used by notifications and gadget to build a common event args. # Returns an array of [args, files] which are ready to be used in the request. - # Returns None if the system isn't ready yet. + # Returns None if we don't have the printer id or octokey yet. def BuildCommonEventArgs(self, event, args=None, progressOverwriteFloat=None, snapshotResizeParams = None, useFinalSnapSnapshot = False): # Ensure we have the required var set already. If not, get out of here. @@ -1112,8 +1112,10 @@ def BuildCommonEventArgs(self, event, args=None, progressOverwriteFloat=None, sn # Get the print info for the current print. pi = PrintInfoManager.Get().GetPrintInfo(self.PrintCookie) if pi is None: + # If we can't get a print info, we can't send the event. + # But return whatever args we have thus far. self.Logger.error("NotificationsHandler failed to get the print info for the current print.") - return args + return [args, None] # Add the required vars args["PrinterId"] = self.PrinterId diff --git a/octoeverywhere/octohttprequest.py b/octoeverywhere/octohttprequest.py index 198e429..8d645bc 100644 --- a/octoeverywhere/octohttprequest.py +++ b/octoeverywhere/octohttprequest.py @@ -429,7 +429,10 @@ def MakeHttpCallAttempt(logger, attemptName, method, url, headers, data, mainRes # We have more fallbacks to try. # Return false so we keep going, but also return this response if we had one. This lets # use capture the main result object, so we can use it eventually if all fallbacks fail. - return OctoHttpRequest.AttemptResult(False, OctoHttpRequest.Result(response.status_code, response.headers, url, isFallback, requestLibResponseObj=response)) + result = None + if response is not None: + OctoHttpRequest.Result(response.status_code, response.headers, url, isFallback, requestLibResponseObj=response) + return OctoHttpRequest.AttemptResult(False, result) # We don't have another fallback, so we need to end this. if mainResult is not None: diff --git a/octoeverywhere/sentry.py b/octoeverywhere/sentry.py index eb18532..4d60118 100644 --- a/octoeverywhere/sentry.py +++ b/octoeverywhere/sentry.py @@ -7,26 +7,38 @@ from sentry_sdk.integrations.threading import ThreadingIntegration from sentry_sdk import capture_exception +from .exceptions import NoSentryReportException + # A helper class to handle Sentry logic. class Sentry: - logger = None - isDevMode = False - lastErrorReport = time.time() - lastErrorCount = 0 + # Holds the process logger. + Logger:logging.Logger = None + + # Flags to help Sentry get setup. + IsSentrySetup:bool = False + isDevMode:bool = False + FilterExceptionsByPackage:bool = False + LastErrorReport:float = time.time() + LastErrorCount:int = 0 - # Sets up Sentry + # This will be called as soon as possible when the process starts to capture the logger, so it's ready for use. @staticmethod - def Init(logger:logging.Logger, versionString:str, distType:str, isDevMode:bool): - # Capture the logger for future use. - Sentry.logger = logger + def SetLogger(logger:logging.Logger): + Sentry.Logger = logger + + # This actually setups sentry. + # It's only called after the plugin version is known, and thus it might be a little into the process lifetime. + @staticmethod + def Setup(versionString:str, distType:str = "unknown", isDevMode:bool = False, enableProfiling:bool = False, filterExceptionsByPackage:bool = False): # Set the dev mode flag. - Sentry.isDevMode = isDevMode + Sentry.IsDevMode = isDevMode + Sentry.FilterExceptionsByPackage = filterExceptionsByPackage # Only setup sentry if we aren't in dev mode. - if Sentry.isDevMode is False: + if Sentry.IsDevMode is False: try: # We don't want sentry to capture error logs, which is it's default. # We do want the logging for breadcrumbs, so we will leave it enabled. @@ -44,81 +56,102 @@ def Init(logger:logging.Logger, versionString:str, distType:str, isDevMode:bool) release=versionString, dist=distType, before_send=Sentry._beforeSendFilter, - traces_sample_rate=0.5, - profiles_sample_rate=0.1 + # This means we will send 100% of errors, maybe we want to reduce this in the future? + sample_rate=1.0, + # Only enable these if we enable profiling. We can't do it in OctoPrint, because it picks up a lot of OctoPrint functions. + traces_sample_rate=0.001 if enableProfiling else 0.0, + profiles_sample_rate=0.01 if enableProfiling else 0.0 ) except Exception as e: - logger.error("Failed to init Sentry: "+str(e)) + if Sentry.Logger is not None: + Sentry.Logger.error("Failed to init Sentry: "+str(e)) + + # Set that sentry is ready to use. + Sentry.IsSentrySetup = True @staticmethod def _beforeSendFilter(event, hint): + + # If we want to filter by package, do it now. + if Sentry.FilterExceptionsByPackage: + # Since all OctoPrint plugins run in the same process, sentry will pick-up unhandled exceptions + # from all kinds of sources. To prevent that from spamming us, if we can pull out a call stack, we will only + # send things that have some origin in our code. This can be any file in the stack or any module with our name in it. + # Otherwise, we will ignore it. + exc_info = hint.get("exc_info") + if exc_info is None or len(exc_info) < 2 or hasattr(exc_info[2], "tb_frame") is False: + Sentry.Logger.error("Failed to extract exception stack in sentry before send.") + return None + + # Check the stack + shouldSend = False + try: + stack = traceback.extract_stack((exc_info[2]).tb_frame) + for s in stack: + # Check for any "octoeverywhere" or "linux_host" in the filename. + # This will match one of the main modules in our code, but exclude any 3rd party code. + filenameLower = s.filename.lower() + if "octoeverywhere" in filenameLower or "linux_host" in filenameLower or "py_installer" in filenameLower: + # If found, return the event so it's reported. + shouldSend = True + break + except Exception as e: + Sentry.Logger.error("Failed to extract exception stack in sentry before send. "+str(e)) + + # If we shouldn't send, then return None to prevent it. + if shouldSend is False: + return None + # To prevent spamming, don't allow clients to send errors too quickly. - # We will simply only allows up to 5 errors reported every 24h. - timeSinceErrorSec = time.time() - Sentry.lastErrorReport - if timeSinceErrorSec < 60 * 60 * 24: - if Sentry.lastErrorCount > 5: + # We will simply only allows up to 5 errors reported every 4h. + timeSinceErrorSec = time.time() - Sentry.LastErrorReport + if timeSinceErrorSec < 60 * 60 * 4: + if Sentry.LastErrorCount > 5: return None else: # A new time window has been entered. - Sentry.lastErrorReport = time.time() - Sentry.lastErrorCount = 0 + Sentry.LastErrorReport = time.time() + Sentry.LastErrorCount = 0 # Increment the report counter - Sentry.lastErrorCount += 1 - - # Since all OctoPrint plugins run in the same process, sentry will pick-up unhandled exceptions - # from all kinds of sources. To prevent that from spamming us, if we can pull out a call stack, we will only - # send things that have some origin in our code. This can be any file in the stack or any module with our name in it. - # Otherwise, we will ignore it. - exc_info = hint.get("exc_info") - if exc_info is None or len(exc_info) < 2 or hasattr(exc_info[2], "tb_frame") is False: - return None - - # Check the stack - try: - stack = traceback.extract_stack((exc_info[2]).tb_frame) - for s in stack: - # Check for any "octoeverywhere" or "linux_host" in the filename. - # This will match one of the main modules in our code, but exclude any 3rd party code. - filenameLower = s.filename.lower() - if "octoeverywhere" in filenameLower or "linux_host" in filenameLower or "py_installer" in filenameLower: - # If found, return the event so it's reported. - return event - except Exception as e: - Sentry.logger.error("Failed to extract exception stack in sentry before send. "+str(e)) - - # Return none to prevent sending. - return None + Sentry.LastErrorCount += 1 + + # Return the event to be reported. + return event # Logs and reports an exception. @staticmethod - def Exception(msg, exception): + def Exception(msg:str, exception:Exception): Sentry._handleException(msg, exception, True) # Only logs an exception, without reporting. @staticmethod - def ExceptionNoSend(msg, exception): + def ExceptionNoSend(msg:str, exception:Exception): Sentry._handleException(msg, exception, False) # Does the work @staticmethod - def _handleException(msg, exception, sendException): + def _handleException(msg:str, exception:Exception, sendException:bool): # This could be called before the class has been inited, in such a case just return. - if Sentry.logger is None: + if Sentry.Logger is None: return tb = traceback.format_exc() exceptionClassType = "unknown_type" if exception is not None: exceptionClassType = exception.__class__.__name__ - Sentry.logger.error(msg + "; "+str(exceptionClassType)+" Exception: " + str(exception) + "; "+str(tb)) + Sentry.Logger.error(msg + "; "+str(exceptionClassType)+" Exception: " + str(exception) + "; "+str(tb)) + + # We have a special exception that we can throw but we won't report it to sentry. + # See the class for details. + if isinstance(exception, NoSentryReportException): + return - # Sentry is disabled for now. # Never send in dev mode, as Sentry will not be setup. - if sendException and Sentry.isDevMode is False: + if Sentry.IsSentrySetup and sendException and Sentry.IsDevMode is False: capture_exception(exception) diff --git a/octoeverywhere/webcamhelper.py b/octoeverywhere/webcamhelper.py index a90d566..9121275 100644 --- a/octoeverywhere/webcamhelper.py +++ b/octoeverywhere/webcamhelper.py @@ -1,6 +1,7 @@ import logging import os import json +import urllib3 from .sentry import Sentry from .octohttprequest import OctoHttpRequest @@ -361,6 +362,12 @@ def _GetSnapshotFromStream(self, url) -> OctoHttpRequest.Result: return None else: Sentry.Exception("Failed to get fallback snapshot due to ConnectionError", e) + except urllib3.exceptions.ProtocolError as e: + if "IncompleteRead" in str(e): + self.Logger.debug("_GetSnapshotFromStream got a incomplete read while reading the stream.") + return None + else: + Sentry.Exception("Failed to get fallback snapshot due to ProtocolError", e) except Exception as e: Sentry.Exception("Failed to get fallback snapshot.", e) diff --git a/octoeverywhere/websocketimpl.py b/octoeverywhere/websocketimpl.py index 56ea3ee..8982c0b 100644 --- a/octoeverywhere/websocketimpl.py +++ b/octoeverywhere/websocketimpl.py @@ -155,3 +155,21 @@ def SendWithOptCode(self, msgBytes, opcode): # If any exception happens during sending, we want to report the error # and shutdown the entire websocket. self.handleWsError(e) + + + # A helper for dealing with common websocket connection exceptions. + @staticmethod + def IsCommonConnectionException(e:Exception): + try: + # This means a device was at the IP, but the port isn't open. + if isinstance(e, ConnectionRefusedError): + return True + # This means the IP doesn't route to a device. + if isinstance(e, OSError) and "No route to host" in str(e): + return True + # This means the other side never responded. + if isinstance(e, TimeoutError) and "Connection timed out" in str(e): + return True + except Exception: + pass + return False diff --git a/octoprint_octoeverywhere/__init__.py b/octoprint_octoeverywhere/__init__.py index 9e4be01..a5c67ab 100644 --- a/octoprint_octoeverywhere/__init__.py +++ b/octoprint_octoeverywhere/__init__.py @@ -148,7 +148,9 @@ def on_startup(self, host, port): self._logger.info("OctoPrint host:" +str(self.OctoPrintLocalHost) + " port:" + str(self.OctoPrintLocalPort)) # Setup Sentry to capture issues. - Sentry.Init(self._logger, self._plugin_version, "octoprint", False) + # We can't enable tracing or profiling in OctoPrint, because it picks up a lot of OctoPrint functions. + Sentry.SetLogger(self._logger) + Sentry.Setup(self._plugin_version, "octoprint", isDevMode=False, enableProfiling=False, filterExceptionsByPackage=True) # Setup our telemetry class. Telemetry.Init(self._logger) diff --git a/octoprint_octoeverywhere/__main__.py b/octoprint_octoeverywhere/__main__.py index 344264c..e56363a 100644 --- a/octoprint_octoeverywhere/__main__.py +++ b/octoprint_octoeverywhere/__main__.py @@ -163,7 +163,8 @@ def GeneratePrinterId(): Compat.SetIsOctoPrint(True) # Init Sentry, but it won't report since we are in dev mode. - Sentry.Init(logger, "0.0.0", "dev", True) + Sentry.SetLogger(logger) + Sentry.Setup("0.0.0", "dev", True, False) Telemetry.Init(logger) if LocalServerAddress is not None: Telemetry.SetServerProtocolAndDomain("http://"+LocalServerAddress) diff --git a/setup.py b/setup.py index 60d8f31..c655062 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this is also parsed by the moonraker module to pull the version, so the string and format must remain the same! -plugin_version = "3.0.3" +plugin_version = "3.0.4" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module @@ -82,7 +82,7 @@ "rsa>=4.9", "dnspython>=2.3.0", "httpx>=0.24.1,<0.26.0", - "urllib3>=1.26.18,<2.0.0" + "urllib3>=1.26.18,<2.0.0", "sentry-sdk>=1.19.1,<2" ] From 703891639e27a7a5a4d88b108a65dba687dcb5eb Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Tue, 12 Mar 2024 16:37:41 -0700 Subject: [PATCH 047/328] Minor fix for the requiremnets. --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 8681e38..e8ecf21 100755 --- a/install.sh +++ b/install.sh @@ -269,7 +269,7 @@ install_or_update_python_env() # Finally, ensure our plugin requirements are installed and updated. log_info "Installing or updating required python libs..." - "${OE_ENV}"/bin/pip3 install -q -r --require-virtualenv --no-cache-dir "${OE_REPO_DIR}"/requirements.txt + "${OE_ENV}"/bin/pip3 install --require-virtualenv --no-cache-dir -q -r "${OE_REPO_DIR}"/requirements.txt log_info "Python libs installed." } From 194fba10f876d3e18c542fe0ff45f1325832d1bb Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Wed, 13 Mar 2024 17:55:10 -0700 Subject: [PATCH 048/328] Updating a few more Sentry things. --- octoeverywhere/sentry.py | 5 +++-- setup.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/octoeverywhere/sentry.py b/octoeverywhere/sentry.py index 4d60118..faba266 100644 --- a/octoeverywhere/sentry.py +++ b/octoeverywhere/sentry.py @@ -48,12 +48,13 @@ def Setup(versionString:str, distType:str = "unknown", isDevMode:bool = False, e ) # Setup and init sentry_sdk.init( - dsn="https://5ce4e93a61f09e32634ab4ffc7a865c0@oe-sentry.octoeverywhere.com/6", + dsn="https://883879bfa2402df86c098f6527f96bfa@oe-sentry.octoeverywhere.com/4", integrations=[ sentry_logging, ThreadingIntegration(propagate_hub=True), ], - release=versionString, + # This is the recommended format + release=f"oe-plugin@{versionString}", dist=distType, before_send=Sentry._beforeSendFilter, # This means we will send 100% of errors, maybe we want to reduce this in the future? diff --git a/setup.py b/setup.py index c655062..6d92e20 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this is also parsed by the moonraker module to pull the version, so the string and format must remain the same! -plugin_version = "3.0.4" +plugin_version = "3.0.5" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 5a30687ee82e6442e827092718d528133e78da5d Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Wed, 13 Mar 2024 18:24:38 -0700 Subject: [PATCH 049/328] More Sentry stuff --- octoeverywhere/sentry.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/octoeverywhere/sentry.py b/octoeverywhere/sentry.py index faba266..e4322cd 100644 --- a/octoeverywhere/sentry.py +++ b/octoeverywhere/sentry.py @@ -48,20 +48,22 @@ def Setup(versionString:str, distType:str = "unknown", isDevMode:bool = False, e ) # Setup and init sentry_sdk.init( - dsn="https://883879bfa2402df86c098f6527f96bfa@oe-sentry.octoeverywhere.com/4", - integrations=[ + dsn= "https://883879bfa2402df86c098f6527f96bfa@oe-sentry.octoeverywhere.com/4", + integrations= [ sentry_logging, ThreadingIntegration(propagate_hub=True), ], # This is the recommended format - release=f"oe-plugin@{versionString}", - dist=distType, - before_send=Sentry._beforeSendFilter, + release= f"oe-plugin@{versionString}", + dist= distType, + environment= "dev" if isDevMode else "production", + before_send= Sentry._beforeSendFilter, # This means we will send 100% of errors, maybe we want to reduce this in the future? - sample_rate=1.0, + enable_tracing= enableProfiling, + sample_rate= 1.0, # Only enable these if we enable profiling. We can't do it in OctoPrint, because it picks up a lot of OctoPrint functions. - traces_sample_rate=0.001 if enableProfiling else 0.0, - profiles_sample_rate=0.01 if enableProfiling else 0.0 + traces_sample_rate= 0.01 if enableProfiling else 0.0, + profiles_sample_rate= 0.01 if enableProfiling else 0.0 ) except Exception as e: if Sentry.Logger is not None: From 5e16ee640513135d804386efe21d21c817fc9925 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Thu, 14 Mar 2024 05:29:58 -0700 Subject: [PATCH 050/328] Fixing more errors from sentry! --- moonraker_octoeverywhere/moonrakerclient.py | 55 +++++++++++++++------ octoeverywhere/websocketimpl.py | 3 ++ setup.py | 2 +- 3 files changed, 44 insertions(+), 16 deletions(-) diff --git a/moonraker_octoeverywhere/moonrakerclient.py b/moonraker_octoeverywhere/moonrakerclient.py index 397b4fa..f9ce566 100644 --- a/moonraker_octoeverywhere/moonrakerclient.py +++ b/moonraker_octoeverywhere/moonrakerclient.py @@ -178,24 +178,49 @@ def GetMoonrakerHostAndPortFromConfig(self): self.Logger.error("GetMoonrakerHostAndPortFromConfig failed to find moonraker config file.") return (currentHostStr, currentPortInt) - # Open and read the config. - moonrakerConfig = configparser.ConfigParser() - moonrakerConfig.read(self.MoonrakerConfigFilePath) + # Ideally we use Config parser + try: + # Open and read the config. + # Set strict to false, which allows for some common errors like duplicate keys to be ignored. + moonrakerConfig = configparser.ConfigParser(allow_no_value=True, strict=False) + moonrakerConfig.read(self.MoonrakerConfigFilePath) + + # We have found that some users don't have a [server] block, so if they don't, return the defaults. + if "server" not in moonrakerConfig: + self.Logger.info("No server block found in the moonraker config, so we are returning the defaults. Host:"+currentHostStr+" Port:"+str(currentPortInt)) + return (currentHostStr, currentPortInt) - # We have found that some users don't have a [server] block, so if they don't, return the defaults. - if "server" not in moonrakerConfig: - self.Logger.info("No server block found in the moonraker config, so we are returning the defaults. Host:"+currentHostStr+" Port:"+str(currentPortInt)) - return (currentHostStr, currentPortInt) + # Otherwise, parse the host and port, if they exist. + serverBlock = moonrakerConfig["server"] + if "host" in serverBlock: + currentHostStr = moonrakerConfig['server']['host'] + if "port" in serverBlock: + currentPortInt = int(moonrakerConfig['server']['port']) - # Otherwise, parse the host and port, if they exist. - serverBlock = moonrakerConfig["server"] - if "host" in serverBlock: - currentHostStr = moonrakerConfig['server']['host'] - if "port" in serverBlock: - currentPortInt = int(moonrakerConfig['server']['port']) + # Done! + return (currentHostStr, currentPortInt) + except configparser.ParsingError as e: + self.Logger.warn("Failed to parse moonraker config file. We will try a manual parse. "+str(e)) + + # If we got here, we failed to parse the file, so we will try to read it manually. + # It's better to get something rather than nothing. + with open(self.MoonrakerConfigFilePath, 'r', encoding="utf-8") as f: + foundHost = False + foundPort = False + # Just look for the host and port lines. + lines = f.readlines() + for l in lines: + lLower = l.lower() + if "host:" in lLower: + currentHostStr = l.split(":", 1)[1].strip() + foundHost = True + if "port:" in lLower: + currentPortInt = int(l.split(":", 1)[1].strip()) + foundPort = True + if foundHost and foundPort: + break + return (currentHostStr, currentPortInt) - # Done! - return (currentHostStr, currentPortInt) except configparser.ParsingError as e: if "Source contains parsing errors" in str(e): self.Logger.error("Failed to parse moonraker config file. "+str(e)) diff --git a/octoeverywhere/websocketimpl.py b/octoeverywhere/websocketimpl.py index 8982c0b..6e1815f 100644 --- a/octoeverywhere/websocketimpl.py +++ b/octoeverywhere/websocketimpl.py @@ -170,6 +170,9 @@ def IsCommonConnectionException(e:Exception): # This means the other side never responded. if isinstance(e, TimeoutError) and "Connection timed out" in str(e): return True + # This just means the server closed the socket. + if isinstance(e, websocket.WebSocketConnectionClosedException) and "Connection to remote host was lost." in str(e): + return True except Exception: pass return False diff --git a/setup.py b/setup.py index 6d92e20..dd80db4 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this is also parsed by the moonraker module to pull the version, so the string and format must remain the same! -plugin_version = "3.0.5" +plugin_version = "3.0.6" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 92638be584d149ec02c0655edf9b351a82540a76 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Thu, 14 Mar 2024 05:56:34 -0700 Subject: [PATCH 051/328] Fixing a minor bug for Bambu. --- bambu_octoeverywhere/bambumodels.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bambu_octoeverywhere/bambumodels.py b/bambu_octoeverywhere/bambumodels.py index 5dd25c4..e222a13 100644 --- a/bambu_octoeverywhere/bambumodels.py +++ b/bambu_octoeverywhere/bambumodels.py @@ -68,7 +68,7 @@ def GetContinuousTimeRemainingSec(self) -> int: if self.gcode_state == "SLICING" or self.gcode_state == "PREPARE": # Reset the last wall clock time to now, so when we transition to running, we don't snap to a strange offset. self.LastTimeRemainingWallClock = time.time() - return self.mc_remaining_time * 60.0 + return int(self.mc_remaining_time * 60) # Compute the time based on when the value last updated. return int(max(0, (self.mc_remaining_time * 60) - (time.time() - self.LastTimeRemainingWallClock))) diff --git a/setup.py b/setup.py index dd80db4..fe3eed8 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this is also parsed by the moonraker module to pull the version, so the string and format must remain the same! -plugin_version = "3.0.6" +plugin_version = "3.0.7" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 14847aaf8400640b4334525ba905ed49594036d7 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Thu, 14 Mar 2024 20:51:35 -0700 Subject: [PATCH 052/328] More Sentry fixes! --- bambu_octoeverywhere/bambuhost.py | 3 +++ moonraker_octoeverywhere/moonrakerhost.py | 3 +++ .../WebStream/octowebstreamwshelper.py | 19 ++++++++++--------- octoeverywhere/notificationshandler.py | 6 ++++-- octoeverywhere/sentry.py | 11 ++++++++--- octoeverywhere/websocketimpl.py | 6 ++++-- octoprint_octoeverywhere/__init__.py | 3 +++ 7 files changed, 35 insertions(+), 16 deletions(-) diff --git a/bambu_octoeverywhere/bambuhost.py b/bambu_octoeverywhere/bambuhost.py index 2bb8efa..2409dfa 100644 --- a/bambu_octoeverywhere/bambuhost.py +++ b/bambu_octoeverywhere/bambuhost.py @@ -83,6 +83,9 @@ def RunBlocking(self, configPath, localStorageDir, repoRoot, devConfig_CanBeNone printerId = self.GetPrinterId() privateKey = self.GetPrivateKey() + # Set the printer ID into sentry. + Sentry.SetPrinterId(printerId) + # Unpack any dev vars that might exist DevLocalServerAddress_CanBeNone = self.GetDevConfigStr(devConfig_CanBeNone, "LocalServerAddress") if DevLocalServerAddress_CanBeNone is not None: diff --git a/moonraker_octoeverywhere/moonrakerhost.py b/moonraker_octoeverywhere/moonrakerhost.py index 5b80f9b..12007da 100644 --- a/moonraker_octoeverywhere/moonrakerhost.py +++ b/moonraker_octoeverywhere/moonrakerhost.py @@ -107,6 +107,9 @@ def RunBlocking(self, klipperConfigDir, isCompanionMode, localStorageDir, servic printerId = self.GetPrinterId() privateKey = self.GetPrivateKey() + # Set the printer id to Sentry. + Sentry.SetPrinterId(printerId) + # Unpack any dev vars that might exist DevLocalServerAddress_CanBeNone = self.GetDevConfigStr(devConfig_CanBeNone, "LocalServerAddress") if DevLocalServerAddress_CanBeNone is not None: diff --git a/octoeverywhere/WebStream/octowebstreamwshelper.py b/octoeverywhere/WebStream/octowebstreamwshelper.py index 26cfa83..3797a53 100644 --- a/octoeverywhere/WebStream/octowebstreamwshelper.py +++ b/octoeverywhere/WebStream/octowebstreamwshelper.py @@ -241,13 +241,6 @@ def IncomingServerMessage(self, webStreamMsg): # Sleep for 5ms. time.sleep(0.005) - # If the websocket object is closed ignore this message. It will throw if the socket is closed - # which will take down the entire OctoStream. But since it's closed the web stream is already cleaning up. - # This can happen if the socket closes locally and we sent the message to clean up to the service, but there - # were already inbound messages on the way. - if self.IsWsObjClosed: - return True - # Note it's ok for this to be empty. Since DataAsByteArray returns 0 if it doesn't # exist, we need to check for it. buffer = webStreamMsg.DataAsByteArray() @@ -272,8 +265,16 @@ def IncomingServerMessage(self, webStreamMsg): else: raise Exception("Web stream ws was sent a data type that's unknown. "+str(msgType)) - # Send! - self.Ws.SendWithOptCode(buffer, sendType) + # Before we send, make sure we have a local websocket still and it's not closed. + # If the websocket object is closed ignore this message. It will throw if the socket is closed + # which will take down the entire OctoStream. But since it's closed the web stream is already cleaning up. + # This can happen if the socket closes locally and we sent the message to clean up to the service, but there + # were already inbound messages on the way. + localWs = self.Ws + if self.IsWsObjClosed or self.IsClosed or localWs is None: + return True + # Send using the known non-null local ws object. + localWs.SendWithOptCode(buffer, sendType) # Log for perf tracking if self.FirstWsMessageSentToLocal is False: diff --git a/octoeverywhere/notificationshandler.py b/octoeverywhere/notificationshandler.py index 490e0dc..56b0a7d 100644 --- a/octoeverywhere/notificationshandler.py +++ b/octoeverywhere/notificationshandler.py @@ -1109,13 +1109,16 @@ def BuildCommonEventArgs(self, event, args=None, progressOverwriteFloat=None, sn if args is None: args = {} + # Define files so we can return an empty dict on any failures. + files = {} + # Get the print info for the current print. pi = PrintInfoManager.Get().GetPrintInfo(self.PrintCookie) if pi is None: # If we can't get a print info, we can't send the event. # But return whatever args we have thus far. self.Logger.error("NotificationsHandler failed to get the print info for the current print.") - return [args, None] + return [args, files] # Add the required vars args["PrinterId"] = self.PrinterId @@ -1153,7 +1156,6 @@ def BuildCommonEventArgs(self, event, args=None, progressOverwriteFloat=None, sn args["DurationSec"] = str(self.GetCurrentDurationSecFloat()) # Also always include a snapshot if we can get one. - files = {} snapshot = None # If we are requested to use a final snapshot, try to use the snapshot from it. diff --git a/octoeverywhere/sentry.py b/octoeverywhere/sentry.py index e4322cd..212cae7 100644 --- a/octoeverywhere/sentry.py +++ b/octoeverywhere/sentry.py @@ -17,7 +17,7 @@ class Sentry: # Flags to help Sentry get setup. IsSentrySetup:bool = False - isDevMode:bool = False + IsDevMode:bool = False FilterExceptionsByPackage:bool = False LastErrorReport:float = time.time() LastErrorCount:int = 0 @@ -32,7 +32,7 @@ def SetLogger(logger:logging.Logger): # This actually setups sentry. # It's only called after the plugin version is known, and thus it might be a little into the process lifetime. @staticmethod - def Setup(versionString:str, distType:str = "unknown", isDevMode:bool = False, enableProfiling:bool = False, filterExceptionsByPackage:bool = False): + def Setup(versionString:str, distType:str, isDevMode:bool = False, enableProfiling:bool = False, filterExceptionsByPackage:bool = False): # Set the dev mode flag. Sentry.IsDevMode = isDevMode Sentry.FilterExceptionsByPackage = filterExceptionsByPackage @@ -63,7 +63,7 @@ def Setup(versionString:str, distType:str = "unknown", isDevMode:bool = False, e sample_rate= 1.0, # Only enable these if we enable profiling. We can't do it in OctoPrint, because it picks up a lot of OctoPrint functions. traces_sample_rate= 0.01 if enableProfiling else 0.0, - profiles_sample_rate= 0.01 if enableProfiling else 0.0 + profiles_sample_rate= 0.01 if enableProfiling else 0.0, ) except Exception as e: if Sentry.Logger is not None: @@ -73,6 +73,11 @@ def Setup(versionString:str, distType:str = "unknown", isDevMode:bool = False, e Sentry.IsSentrySetup = True + @staticmethod + def SetPrinterId(printerId:str): + sentry_sdk.set_context("octoeverywhere", { "printer-id": printerId }) + + @staticmethod def _beforeSendFilter(event, hint): diff --git a/octoeverywhere/websocketimpl.py b/octoeverywhere/websocketimpl.py index 6e1815f..36807a2 100644 --- a/octoeverywhere/websocketimpl.py +++ b/octoeverywhere/websocketimpl.py @@ -170,8 +170,10 @@ def IsCommonConnectionException(e:Exception): # This means the other side never responded. if isinstance(e, TimeoutError) and "Connection timed out" in str(e): return True - # This just means the server closed the socket. - if isinstance(e, websocket.WebSocketConnectionClosedException) and "Connection to remote host was lost." in str(e): + # This just means the server closed the socket, + # or the socket connection was lost after a long delay + # or there was a DNS name resolve failure. + if isinstance(e, websocket.WebSocketConnectionClosedException) and ("Connection to remote host was lost." in str(e) or "ping/pong timed out" in str(e) or "Name or service not known" in str(e)): return True except Exception: pass diff --git a/octoprint_octoeverywhere/__init__.py b/octoprint_octoeverywhere/__init__.py index a5c67ab..9cc78a7 100644 --- a/octoprint_octoeverywhere/__init__.py +++ b/octoprint_octoeverywhere/__init__.py @@ -167,6 +167,9 @@ def on_startup(self, host, port): # Ensure the plugin version is updated in the settings for the frontend. self.EnsurePluginVersionSet() + # Set the printer id to Sentry. + Sentry.SetPrinterId(printerId) + # Init the static local auth helper LocalAuth.Init(self._logger, self._user_manager) From ce514c4f263d7aa17d096fe767b9cec0ed8a38c4 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Thu, 14 Mar 2024 21:06:22 -0700 Subject: [PATCH 053/328] More Sentry bug fixes! --- moonraker_octoeverywhere/moonrakerwebcamhelper.py | 2 +- moonraker_octoeverywhere/uiinjector.py | 9 +++++++-- octoeverywhere/websocketimpl.py | 2 ++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/moonraker_octoeverywhere/moonrakerwebcamhelper.py b/moonraker_octoeverywhere/moonrakerwebcamhelper.py index 8db1b13..5436b82 100644 --- a/moonraker_octoeverywhere/moonrakerwebcamhelper.py +++ b/moonraker_octoeverywhere/moonrakerwebcamhelper.py @@ -447,7 +447,7 @@ def _TryToFigureOutSnapshotUrl(self, streamUrl): streamUrlLower = streamUrl.lower() c_streamAction = "action=stream" c_snapshotAction = "action=snapshot" - indexOfStreamSuffix = streamUrlLower.index(c_streamAction) + indexOfStreamSuffix = streamUrlLower.find(c_streamAction) if indexOfStreamSuffix != -1: # We found the action=stream, try replacing it and see if we hit a valid endpoint. diff --git a/moonraker_octoeverywhere/uiinjector.py b/moonraker_octoeverywhere/uiinjector.py index 1e58c16..ac20e58 100644 --- a/moonraker_octoeverywhere/uiinjector.py +++ b/moonraker_octoeverywhere/uiinjector.py @@ -261,8 +261,13 @@ def _InjectIntoHtml(self, indexHtmlFilePath) -> bool: return False # Write the file back. - with open(indexHtmlFilePath, 'w', encoding="utf-8") as f: - f.write(htmlText) + # If we can't write, it's ok. + try: + with open(indexHtmlFilePath, 'w', encoding="utf-8") as f: + f.write(htmlText) + except PermissionError as e: + self.Logger.warn(f"Failed to write to {indexHtmlFilePath}, permission error. This is ok. "+str(e)) + return False self.Logger.info("No existing ui tags found, so we added them") return True diff --git a/octoeverywhere/websocketimpl.py b/octoeverywhere/websocketimpl.py index 36807a2..f91c297 100644 --- a/octoeverywhere/websocketimpl.py +++ b/octoeverywhere/websocketimpl.py @@ -170,6 +170,8 @@ def IsCommonConnectionException(e:Exception): # This means the other side never responded. if isinstance(e, TimeoutError) and "Connection timed out" in str(e): return True + if isinstance(e, websocket.WebSocketTimeoutException): + return True # This just means the server closed the socket, # or the socket connection was lost after a long delay # or there was a DNS name resolve failure. From 6ab57d10ae581504519185ba340106ca2ea7f6ef Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Fri, 15 Mar 2024 20:24:48 -0700 Subject: [PATCH 054/328] Fixing more bugs from sentry! --- .../systemconfigmanager.py | 11 ++-- octoeverywhere/notificationshandler.py | 26 +++++----- octoeverywhere/sentry.py | 52 +++++++++++++------ octoeverywhere/websocketimpl.py | 4 +- setup.py | 2 +- 5 files changed, 61 insertions(+), 34 deletions(-) diff --git a/moonraker_octoeverywhere/systemconfigmanager.py b/moonraker_octoeverywhere/systemconfigmanager.py index 8097e56..3a654c4 100644 --- a/moonraker_octoeverywhere/systemconfigmanager.py +++ b/moonraker_octoeverywhere/systemconfigmanager.py @@ -144,9 +144,14 @@ def _ensureMoonrakerConfigHasUpdateConfigInclude(klipperConfigDir, logger): logger.info("Our existing update config file include was found in the moonraker config.") return - # The text wasn't found, append it to the end. - with open(moonrakerConfigFilePath,'a', encoding="utf-8") as f: - f.write("\n"+includeText+"\n") + # We should always have permissions, since the installer sets them, but if not, we'll just fail. + try: + # The text wasn't found, append it to the end of the config file. + with open(moonrakerConfigFilePath, 'a', encoding="utf-8") as f: + f.write("\n"+includeText+"\n") + except PermissionError as e: + logger.error("We tried to update the moonraker config to add our include, but we don't have file permissions. "+str(e)) + return logger.info("Our update config include was not found in the moonraker config, so we added it.") diff --git a/octoeverywhere/notificationshandler.py b/octoeverywhere/notificationshandler.py index 56b0a7d..a297de2 100644 --- a/octoeverywhere/notificationshandler.py +++ b/octoeverywhere/notificationshandler.py @@ -1020,7 +1020,7 @@ def _getCurrentProgressFloat(self): # Sends the event # Returns True on success, otherwise False - def _sendEvent(self, event, args = None, progressOverwriteFloat = None, useFinalSnapSnapshot = False): + def _sendEvent(self, event:str, args = None, progressOverwriteFloat = None, useFinalSnapSnapshot = False): # Push the work off to a thread so we don't hang OctoPrint's plugin callbacks. thread = threading.Thread(target=self._sendEventThreadWorker, args=(event, args, progressOverwriteFloat, useFinalSnapSnapshot, )) thread.start() @@ -1030,7 +1030,7 @@ def _sendEvent(self, event, args = None, progressOverwriteFloat = None, useFinal # Sends the event # Returns True on success, otherwise False - def _sendEventThreadWorker(self, event, args = None, progressOverwriteFloat = None, useFinalSnapSnapshot = False): + def _sendEventThreadWorker(self, event:str, args = None, progressOverwriteFloat = None, useFinalSnapSnapshot = False): try: # Build the common even args. requestArgs = self.BuildCommonEventArgs(event, args, progressOverwriteFloat=progressOverwriteFloat, useFinalSnapSnapshot=useFinalSnapSnapshot) @@ -1098,8 +1098,9 @@ def _sendEventThreadWorker(self, event, args = None, progressOverwriteFloat = No # Used by notifications and gadget to build a common event args. # Returns an array of [args, files] which are ready to be used in the request. + # The args and files will always contain any information that can be gathered at the time of the call. # Returns None if we don't have the printer id or octokey yet. - def BuildCommonEventArgs(self, event, args=None, progressOverwriteFloat=None, snapshotResizeParams = None, useFinalSnapSnapshot = False): + def BuildCommonEventArgs(self, event:str, args=None, progressOverwriteFloat=None, snapshotResizeParams = None, useFinalSnapSnapshot = False): # Ensure we have the required var set already. If not, get out of here. if self.PrinterId is None or self.OctoKey is None: @@ -1113,24 +1114,21 @@ def BuildCommonEventArgs(self, event, args=None, progressOverwriteFloat=None, sn files = {} # Get the print info for the current print. + # We should always be able to get the print info, but if not, we will still try to send what we can anyways. pi = PrintInfoManager.Get().GetPrintInfo(self.PrintCookie) - if pi is None: - # If we can't get a print info, we can't send the event. - # But return whatever args we have thus far. - self.Logger.error("NotificationsHandler failed to get the print info for the current print.") - return [args, files] + if pi is not None: + args["PrintId"] = pi.GetPrintId() + args["FileName"] = str(pi.GetFileName()) + args["FileSizeKb"] = str(pi.GetFileSizeKBytes()) + args["FilamentUsageMm"] = str(pi.GetEstFilamentUsageMm()) + else: + Sentry.LogError("NotificationsHandler failed to get the print info for the current print.", {"Cookie": self.PrintCookie, "Event": event}) # Add the required vars args["PrinterId"] = self.PrinterId - args["PrintId"] = pi.GetPrintId() args["OctoKey"] = self.OctoKey args["Event"] = event - # Always add the file name and other common props - args["FileName"] = str(pi.GetFileName()) - args["FileSizeKb"] = str(pi.GetFileSizeKBytes()) - args["FilamentUsageMm"] = str(pi.GetEstFilamentUsageMm()) - # Always include the ETA, note this will be -1 if the time is unknown. timeRemainEstStr = str(self.PrinterStateInterface.GetPrintTimeRemainingEstimateInSeconds()) args["TimeRemainingSec"] = timeRemainEstStr diff --git a/octoeverywhere/sentry.py b/octoeverywhere/sentry.py index 212cae7..b5498d2 100644 --- a/octoeverywhere/sentry.py +++ b/octoeverywhere/sentry.py @@ -5,7 +5,6 @@ import sentry_sdk from sentry_sdk.integrations.logging import LoggingIntegration from sentry_sdk.integrations.threading import ThreadingIntegration -from sentry_sdk import capture_exception from .exceptions import NoSentryReportException @@ -13,7 +12,7 @@ class Sentry: # Holds the process logger. - Logger:logging.Logger = None + _Logger:logging.Logger = None # Flags to help Sentry get setup. IsSentrySetup:bool = False @@ -26,7 +25,7 @@ class Sentry: # This will be called as soon as possible when the process starts to capture the logger, so it's ready for use. @staticmethod def SetLogger(logger:logging.Logger): - Sentry.Logger = logger + Sentry._Logger = logger # This actually setups sentry. @@ -66,8 +65,8 @@ def Setup(versionString:str, distType:str, isDevMode:bool = False, enableProfili profiles_sample_rate= 0.01 if enableProfiling else 0.0, ) except Exception as e: - if Sentry.Logger is not None: - Sentry.Logger.error("Failed to init Sentry: "+str(e)) + if Sentry._Logger is not None: + Sentry._Logger.error("Failed to init Sentry: "+str(e)) # Set that sentry is ready to use. Sentry.IsSentrySetup = True @@ -89,7 +88,7 @@ def _beforeSendFilter(event, hint): # Otherwise, we will ignore it. exc_info = hint.get("exc_info") if exc_info is None or len(exc_info) < 2 or hasattr(exc_info[2], "tb_frame") is False: - Sentry.Logger.error("Failed to extract exception stack in sentry before send.") + Sentry._Logger.error("Failed to extract exception stack in sentry before send.") return None # Check the stack @@ -105,7 +104,7 @@ def _beforeSendFilter(event, hint): shouldSend = True break except Exception as e: - Sentry.Logger.error("Failed to extract exception stack in sentry before send. "+str(e)) + Sentry._Logger.error("Failed to extract exception stack in sentry before send. "+str(e)) # If we shouldn't send, then return None to prevent it. if shouldSend is False: @@ -129,31 +128,49 @@ def _beforeSendFilter(event, hint): return event + # Sends an error log to sentry. + # This is useful for debugging things that shouldn't be happening. + @staticmethod + def LogError(msg:str, extras:dict = None) -> None: + if Sentry._Logger is None: + return + Sentry._Logger.error(f"Sentry Error: {msg}") + # Never send in dev mode, as Sentry will not be setup. + if Sentry.IsSentrySetup and Sentry.IsDevMode is False: + with sentry_sdk.push_scope() as scope: + scope.set_level("error") + if extras is not None: + for key, value in extras.items(): + scope.set_extra(key, value) + sentry_sdk.capture_message(msg) + + # Logs and reports an exception. + # If there's no exception, use LogError instead. @staticmethod - def Exception(msg:str, exception:Exception): - Sentry._handleException(msg, exception, True) + def Exception(msg:str, exception:Exception, extras:dict = None): + Sentry._handleException(msg, exception, True, extras) # Only logs an exception, without reporting. @staticmethod - def ExceptionNoSend(msg:str, exception:Exception): - Sentry._handleException(msg, exception, False) + def ExceptionNoSend(msg:str, exception:Exception, extras:dict = None): + Sentry._handleException(msg, exception, False, extras) # Does the work @staticmethod - def _handleException(msg:str, exception:Exception, sendException:bool): + def _handleException(msg:str, exception:Exception, sendException:bool, extras:dict = None): # This could be called before the class has been inited, in such a case just return. - if Sentry.Logger is None: + if Sentry._Logger is None: return tb = traceback.format_exc() exceptionClassType = "unknown_type" if exception is not None: exceptionClassType = exception.__class__.__name__ - Sentry.Logger.error(msg + "; "+str(exceptionClassType)+" Exception: " + str(exception) + "; "+str(tb)) + Sentry._Logger.error(msg + "; "+str(exceptionClassType)+" Exception: " + str(exception) + "; "+str(tb)) # We have a special exception that we can throw but we won't report it to sentry. # See the class for details. @@ -162,4 +179,9 @@ def _handleException(msg:str, exception:Exception, sendException:bool): # Never send in dev mode, as Sentry will not be setup. if Sentry.IsSentrySetup and sendException and Sentry.IsDevMode is False: - capture_exception(exception) + with sentry_sdk.push_scope() as scope: + scope.set_extra("Exception Message", msg) + if extras is not None: + for key, value in extras.items(): + scope.set_extra(key, value) + sentry_sdk.capture_exception(exception) diff --git a/octoeverywhere/websocketimpl.py b/octoeverywhere/websocketimpl.py index f91c297..f919444 100644 --- a/octoeverywhere/websocketimpl.py +++ b/octoeverywhere/websocketimpl.py @@ -164,8 +164,10 @@ def IsCommonConnectionException(e:Exception): # This means a device was at the IP, but the port isn't open. if isinstance(e, ConnectionRefusedError): return True + if isinstance(e, ConnectionResetError): + return True # This means the IP doesn't route to a device. - if isinstance(e, OSError) and "No route to host" in str(e): + if isinstance(e, OSError) and ("No route to host" in str(e) or "Network is unreachable" in str(e)): return True # This means the other side never responded. if isinstance(e, TimeoutError) and "Connection timed out" in str(e): diff --git a/setup.py b/setup.py index fe3eed8..4841b62 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this is also parsed by the moonraker module to pull the version, so the string and format must remain the same! -plugin_version = "3.0.7" +plugin_version = "3.0.8" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 022f47e620b5aebe0835be34152e4043b56960a0 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sat, 16 Mar 2024 16:09:56 -0700 Subject: [PATCH 055/328] Minor changes. --- moonraker_octoeverywhere/systemconfigmanager.py | 9 ++++++--- octoeverywhere/notificationshandler.py | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/moonraker_octoeverywhere/systemconfigmanager.py b/moonraker_octoeverywhere/systemconfigmanager.py index 3a654c4..7a2ee9d 100644 --- a/moonraker_octoeverywhere/systemconfigmanager.py +++ b/moonraker_octoeverywhere/systemconfigmanager.py @@ -121,9 +121,12 @@ def EnsureAllowedServicesFile(logger, klipperConfigDir, serviceName): return # Add our name. - with open(allowedServiceFile,'a', encoding="utf-8") as f: - # The current format this doc is not have a trailing \n, so we need to add one. - f.write("\n"+serviceName) + try: + with open(allowedServiceFile,'a', encoding="utf-8") as f: + # The current format this doc is not have a trailing \n, so we need to add one. + f.write("\n"+serviceName) + except PermissionError as e: + logger.warn("We tried to write the moonraker allowed services file but don't have permissions "+str(e)) logger.info("Our name wasn't found in moonraker's allowed service file, so we added it.") diff --git a/octoeverywhere/notificationshandler.py b/octoeverywhere/notificationshandler.py index a297de2..0a5df44 100644 --- a/octoeverywhere/notificationshandler.py +++ b/octoeverywhere/notificationshandler.py @@ -1113,8 +1113,8 @@ def BuildCommonEventArgs(self, event:str, args=None, progressOverwriteFloat=None # Define files so we can return an empty dict on any failures. files = {} - # Get the print info for the current print. - # We should always be able to get the print info, but if not, we will still try to send what we can anyways. + # Get the print info if there is a current print. + # Remember that some notifications will fire when there's no print running, like if OctoPrint loses it's connection to the printer while idle. pi = PrintInfoManager.Get().GetPrintInfo(self.PrintCookie) if pi is not None: args["PrintId"] = pi.GetPrintId() From df9933e8669deba06e254d02f68fdc89c26e4c9a Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Mon, 18 Mar 2024 20:53:46 -0700 Subject: [PATCH 056/328] Fixing sentry errors and updating the multicam logic! --- .vscode/settings.json | 1 + bambu_octoeverywhere/bambuclient.py | 21 +-- bambu_octoeverywhere/bambuwebcamhelper.py | 4 +- moonraker_octoeverywhere/moonrakerclient.py | 11 ++ .../moonrakercredentailmanager.py | 2 +- .../systemconfigmanager.py | 3 +- .../webrequestresponsehandler.py | 44 ++++-- .../WebStream/octowebstreamhttphelper.py | 2 +- octoeverywhere/commandhandler.py | 32 ++-- octoeverywhere/notificationshandler.py | 7 +- octoeverywhere/octohttprequest.py | 20 ++- octoeverywhere/octosessionimpl.py | 6 +- octoeverywhere/webcamhelper.py | 146 ++++++++++++------ octoeverywhere/websocketimpl.py | 16 +- octoprint_octoeverywhere/__init__.py | 6 +- octoprint_octoeverywhere/slipstream.py | 2 +- setup.py | 2 +- 17 files changed, 224 insertions(+), 101 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 5b1609e..95f4b62 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -183,6 +183,7 @@ "snapshotresizeparams", "Softwareupdate", "somename", + "sooooooo", "Spammy", "sslopt", "Starbound", diff --git a/bambu_octoeverywhere/bambuclient.py b/bambu_octoeverywhere/bambuclient.py index 2aa65ee..331a053 100644 --- a/bambu_octoeverywhere/bambuclient.py +++ b/bambu_octoeverywhere/bambuclient.py @@ -124,16 +124,19 @@ def _ClientWorker(self): # This will run forever, including handling reconnects and such. self.Client.loop_forever() - - except ConnectionRefusedError as e: - # This means there was no open socket at the given IP and port. - self.Logger.error(f"Failed to connect to the Bambu printer {ipOrHostname}:{self.PortStr}, we will retry in a bit. "+str(e)) - except TimeoutError as e: - # This means there was no open socket at the given IP and port. - self.Logger.error(f"Failed to connect to the Bambu printer {ipOrHostname}:{self.PortStr}, we will retry in a bit. "+str(e)) except Exception as e: - # Random other errors. - Sentry.Exception("Failed to connect to Bambu printer.", e) + if isinstance(e, ConnectionRefusedError): + # This means there was no open socket at the given IP and port. + self.Logger.error(f"Failed to connect to the Bambu printer {ipOrHostname}:{self.PortStr}, we will retry in a bit. "+str(e)) + elif isinstance(e, TimeoutError): + # This means there was no open socket at the given IP and port. + self.Logger.error(f"Failed to connect to the Bambu printer {ipOrHostname}:{self.PortStr}, we will retry in a bit. "+str(e)) + elif isinstance(e, OSError) and "Network is unreachable" in str(e): + # This means the IP doesn't route to a device. + self.Logger.error(f"Failed to connect to the Bambu printer {ipOrHostname}:{self.PortStr}, we will retry in a bit. "+str(e)) + else: + # Random other errors. + Sentry.Exception("FaWiled to connect to Bambu printer.", e) # Sleep for a bit between tries. # The main consideration here is to not log too much when the printer is off. But we do still want to connect quickly, when it's back on. diff --git a/bambu_octoeverywhere/bambuwebcamhelper.py b/bambu_octoeverywhere/bambuwebcamhelper.py index 55a4183..016a5d8 100644 --- a/bambu_octoeverywhere/bambuwebcamhelper.py +++ b/bambu_octoeverywhere/bambuwebcamhelper.py @@ -41,7 +41,7 @@ def GetWebcamConfig(self): # On failure, return None # On success, this will return a valid OctoHttpRequest that's fully filled out. # The snapshot will always already be fully read, and will be FullBodyBuffer var. - def GetSnapshot_Override(self, cameraName:str): + def GetSnapshot_Override(self, cameraIndex:int): # Try to get a snapshot from our QuickCam system. img = QuickCam.Get().GetCurrentImage() if img is None: @@ -60,7 +60,7 @@ def GetSnapshot_Override(self, cameraName:str): # On failure, return None # On success, this will return a valid OctoHttpRequest that's fully filled out. # This must return an OctoHttpRequest object with a custom body read stream. - def GetStream_Override(self, cameraName:str): + def GetStream_Override(self, cameraIndex:int): # We must create a new instance of this class per stream to ensure all of the vars stay in it's context # and the streams are cleaned up properly. sm = StreamInstance(self.Logger) diff --git a/moonraker_octoeverywhere/moonrakerclient.py b/moonraker_octoeverywhere/moonrakerclient.py index f9ce566..3b8ca63 100644 --- a/moonraker_octoeverywhere/moonrakerclient.py +++ b/moonraker_octoeverywhere/moonrakerclient.py @@ -7,6 +7,8 @@ import math import configparser +import websocket + from octoeverywhere.compat import Compat from octoeverywhere.sentry import Sentry from octoeverywhere.websocketimpl import Client @@ -201,6 +203,12 @@ def GetMoonrakerHostAndPortFromConfig(self): return (currentHostStr, currentPortInt) except configparser.ParsingError as e: self.Logger.warn("Failed to parse moonraker config file. We will try a manual parse. "+str(e)) + except AttributeError as e: + # This seems to be a configparser bug. + if "'NoneType' object has no attribute 'append'" in str(e): + self.Logger.warn("Failed to parse moonraker config file. We will try a manual parse. "+str(e)) + else: + raise e # If we got here, we failed to parse the file, so we will try to read it manually. # It's better to get something rather than nothing. @@ -722,6 +730,9 @@ def _onWsError(self, ws, exception): if Client.IsCommonConnectionException(exception): # Don't bother logging, this just means there's no server to connect to. pass + elif isinstance(exception, websocket.WebSocketBadStatusException) and "Handshake status" in str(exception): + # This is moonraker specific, we sometimes see stuff like "Handshake status 502 Bad Gateway" + self.Logger.info(f"Failed to connect to moonraker due to bad gateway stats. {exception}") else: Sentry.Exception("Exception rased from moonraker client websocket connection. The connection will be closed.", exception) diff --git a/moonraker_octoeverywhere/moonrakercredentailmanager.py b/moonraker_octoeverywhere/moonrakercredentailmanager.py index 7cd40f2..10e0864 100644 --- a/moonraker_octoeverywhere/moonrakercredentailmanager.py +++ b/moonraker_octoeverywhere/moonrakercredentailmanager.py @@ -55,7 +55,7 @@ def TryToGetApiKey(self) -> str or None: # First, we need to find the unix socket to connect to moonrakerSocketFilePath = self._TryToFindUnixSocket() if moonrakerSocketFilePath is None: - self.Logger.Warn("No moonraker unix socket file could be found.") + self.Logger.warn("No moonraker unix socket file could be found.") return None try: diff --git a/moonraker_octoeverywhere/systemconfigmanager.py b/moonraker_octoeverywhere/systemconfigmanager.py index 7a2ee9d..c86b1fa 100644 --- a/moonraker_octoeverywhere/systemconfigmanager.py +++ b/moonraker_octoeverywhere/systemconfigmanager.py @@ -99,7 +99,7 @@ def EnsureUpdateManagerFilesSetup(logger:logging.Logger, klipperConfigDir, servi # Details: https://moonraker.readthedocs.io/en/latest/configuration/#allowed-services # TODO - Eventually we will get our PR in that will add this to moonraker's default list. @staticmethod - def EnsureAllowedServicesFile(logger, klipperConfigDir, serviceName): + def EnsureAllowedServicesFile(logger, klipperConfigDir, serviceName) -> None: # Make the expected file path, it should be one folder up from the config folder dataRootDir = os.path.abspath(os.path.join(klipperConfigDir, os.pardir)) allowedServiceFile = os.path.join(dataRootDir, "moonraker.asvc") @@ -127,6 +127,7 @@ def EnsureAllowedServicesFile(logger, klipperConfigDir, serviceName): f.write("\n"+serviceName) except PermissionError as e: logger.warn("We tried to write the moonraker allowed services file but don't have permissions "+str(e)) + return logger.info("Our name wasn't found in moonraker's allowed service file, so we added it.") diff --git a/moonraker_octoeverywhere/webrequestresponsehandler.py b/moonraker_octoeverywhere/webrequestresponsehandler.py index fa11972..2c0ec1e 100644 --- a/moonraker_octoeverywhere/webrequestresponsehandler.py +++ b/moonraker_octoeverywhere/webrequestresponsehandler.py @@ -3,6 +3,7 @@ from octoeverywhere.compat import Compat from octoeverywhere.sentry import Sentry +from octoeverywhere.octohttprequest import OctoHttpRequest # The context class we return if we want to handle this request. class ResponseHandlerContext: @@ -61,12 +62,12 @@ def CheckIfResponseNeedsToBeHandled(self, uri:str) -> ResponseHandlerContext: # If we returned a context above in CheckIfResponseNeedsToBeHandled, this will be called after the web request is made # and the body is fully read. The entire body will be read into the bodyBuffer. # We are able to modify the bodyBuffer as we wish or not, but we must return the full bodyBuffer back to be returned. - def HandleResponse(self, contextObject:ResponseHandlerContext, bodyBuffer: bytes) -> bytes: + def HandleResponse(self, contextObject:ResponseHandlerContext, octoHttpResult:OctoHttpRequest.Result, bodyBuffer: bytes) -> bytes: try: if contextObject.Type == ResponseHandlerContext.MainsailConfig: - return self._HandleMainsailConfig(bodyBuffer) + return self._HandleMainsailConfig(octoHttpResult, bodyBuffer) elif contextObject.Type == ResponseHandlerContext.CameraStreamerWebRTCSdp: - return self._HandleWebRtcSdpResponse(bodyBuffer) + return self._HandleWebRtcSdpResponse(octoHttpResult, bodyBuffer) else: self.Logger.Error("MoonrakerWebRequestResponseHandler tired to handle a context with an unknown Type? "+str(contextObject.Type)) except Exception as e: @@ -74,7 +75,7 @@ def HandleResponse(self, contextObject:ResponseHandlerContext, bodyBuffer: bytes return bodyBuffer - def _HandleMainsailConfig(self, bodyBuffer:bytes) -> bytes: + def _HandleMainsailConfig(self, octoHttpResult:OctoHttpRequest.Result, bodyBuffer:bytes) -> bytes: # # Note that we identify this file just by dont a .endsWith("/config.json") to the URL. Thus other things could match it # and we need to be careful to only edit it if we find what we expect. @@ -86,18 +87,29 @@ def _HandleMainsailConfig(self, bodyBuffer:bytes) -> bytes: # Right now we can't do anything else, because moonraker only allows the user to set custom hostname and ports, not paths, to call # the different websockets at. But in the future, we could look into redirecting the websocket and known moonraker http api paths to the # known moonraker instance running with this octoeverywhere instance. - mainsailConfig = json.loads(bodyBuffer.decode("utf8")) - if "instancesDB" in mainsailConfig: - # Set mainsail and be sure to clear our any instances. - mainsailConfig["instancesDB"] = "moonraker" - mainsailConfig["instances"] = [] - # Older versions struggle to connect to the websocket if we don't set this port as well - # We can always set it to 443, because we will always have SSL. - mainsailConfig["port"] = 443 - return json.dumps(mainsailConfig, indent=4).encode("utf8") - - - def _HandleWebRtcSdpResponse(self, bodyBuffer:bytes) -> bytes: + try: + mainsailConfig = json.loads(bodyBuffer.decode("utf8")) + if "instancesDB" in mainsailConfig: + # Set mainsail and be sure to clear our any instances. + mainsailConfig["instancesDB"] = "moonraker" + mainsailConfig["instances"] = [] + # Older versions struggle to connect to the websocket if we don't set this port as well + # We can always set it to 443, because we will always have SSL. + mainsailConfig["port"] = 443 + return json.dumps(mainsailConfig, indent=4).encode("utf8") + except Exception as e: + body = None + try: + body = bodyBuffer.decode("utf8") + except Exception: + pass + Sentry.Exception(f"MainsailConfigHandler exception while handling mainsail config. {octoHttpResult.StatusCode}; body: {body}", e) + # On failure, return the original result. + return bodyBuffer + + + + def _HandleWebRtcSdpResponse(self, octoHttpResult:OctoHttpRequest.Result, bodyBuffer:bytes) -> bytes: # # As of Crowsnest 4.0, it now supports camera-stream (like OctoPrint) which supports WebRTC. # This is an obvious winner in camera streaming, because WebRTC is much better than mjpeg streaming. diff --git a/octoeverywhere/WebStream/octowebstreamhttphelper.py b/octoeverywhere/WebStream/octowebstreamhttphelper.py index ea3682d..b2549ac 100644 --- a/octoeverywhere/WebStream/octowebstreamhttphelper.py +++ b/octoeverywhere/WebStream/octowebstreamhttphelper.py @@ -740,7 +740,7 @@ def readContentFromBodyAndMakeDataVector(self, builder, octoHttpResult:OctoHttpR else: # If we have the compat handler, give it the buffer before we finalize the size, as it might want to edit the buffer. if Compat.HasWebRequestResponseHandler(): - finalDataBuffer = Compat.GetWebRequestResponseHandler().HandleResponse(responseHandlerContext, finalDataBuffer) + finalDataBuffer = Compat.GetWebRequestResponseHandler().HandleResponse(responseHandlerContext, octoHttpResult, finalDataBuffer) # If we were asked to compress, do it originalBufferSize = len(finalDataBuffer) diff --git a/octoeverywhere/commandhandler.py b/octoeverywhere/commandhandler.py index 8a9a767..9b11647 100644 --- a/octoeverywhere/commandhandler.py +++ b/octoeverywhere/commandhandler.py @@ -148,37 +148,51 @@ def GetStatus(self): except Exception as e: Sentry.ExceptionNoSend("API command GetStatus failed to get OctoPrint version", e) + # Get the list webcams response as well + # Don't include the URL to reduce the payload size. + webcamInfoCommandResponse = self.ListWebcams(False) + # This shouldn't be possible, but we will check for sanity sake. + if webcamInfoCommandResponse is None or webcamInfoCommandResponse.StatusCode != 200: + self.Logger.error("GetStatus command failed to get webcam info.") + webcamInfoCommandResponse = CommandResponse.Success({}) + # Build the final response responseObj = { "JobStatus" : jobStatus, "OctoEverywhereStatus" : octoeverywhereStatus, - "PlatformVersion" : versionStr + "PlatformVersion" : versionStr, + "ListWebcams" : webcamInfoCommandResponse.ResultDict } return CommandResponse.Success(responseObj) # Must return a CommandResponse - def ListWebcams(self): + def ListWebcams(self, includeUrls = True): # Get all of the known webcams webcamSettingsItems = WebcamHelper.Get().ListWebcams() if webcamSettingsItems is None: webcamSettingsItems = [] # We need to convert the objects into a dic to serialize. + # Note this format is also used for GetStatus! webcams = [] for i in webcamSettingsItems: wc = {} wc["Name"] = i.Name - wc["SnapshotUrl"] = i.SnapshotUrl - wc["StreamUrl"] = i.StreamUrl wc["FlipH"] = i.FlipH wc["FlipV"] = i.FlipV wc["Rotation"] = i.Rotation + if includeUrls: + wc["SnapshotUrl"] = i.SnapshotUrl + wc["StreamUrl"] = i.StreamUrl webcams.append(wc) - # Build the response + + # We always use the default index, which is a reflection of the current camera list. + # We don't use the name, we only use that internally to keep track of the current index. + defaultIndex = WebcamHelper.Get().GetDefaultCameraIndex(webcamSettingsItems) responseObj = { "Webcams" : webcams, - "DefaultName" : WebcamHelper.Get().GetDefaultCameraName() + "DefaultIndex" : defaultIndex } return CommandResponse.Success(responseObj) @@ -360,18 +374,18 @@ def ProcessCommand(self, commandPath, jsonObj_CanBeNone): class CommandResponse(): @staticmethod - def Success(resultDict): + def Success(resultDict:dict): if resultDict is None: resultDict = {} return CommandResponse(200, resultDict, None) @staticmethod - def Error(statusCode, errorStr_CanBeNull): + def Error(statusCode:int, errorStr_CanBeNull:str): return CommandResponse(statusCode, None, errorStr_CanBeNull) - def __init__(self, statusCode, resultDict, errorStr_CanBeNull): + def __init__(self, statusCode:int, resultDict:dict, errorStr_CanBeNull:str): self.StatusCode = statusCode self.ResultDict = resultDict self.ErrorStr = errorStr_CanBeNull diff --git a/octoeverywhere/notificationshandler.py b/octoeverywhere/notificationshandler.py index 0a5df44..95f8065 100644 --- a/octoeverywhere/notificationshandler.py +++ b/octoeverywhere/notificationshandler.py @@ -898,9 +898,12 @@ def GetNotificationSnapshot(self, snapshotResizeParams = None): buffer.close() else: self.Logger.warn("Can't manipulate image because the Image rotation lib failed to import.") - except Exception as ex: + except Exception as e: # Note that in the case of an exception we don't overwrite the original snapshot buffer, so something can still be sent. - Sentry.ExceptionNoSend("Failed to manipulate image for notifications", ex) + if "name 'Image' is not defined" in str(e): + self.Logger.info("Can't manipulate image because the Image rotation lib failed to import.") + else: + Sentry.Exception("Failed to manipulate image for notifications", e) # Ensure in the end, the snapshot is a reasonable size. if len(snapshot) > NotificationsHandler.MaxSnapshotFileSizeBytes: diff --git a/octoeverywhere/octohttprequest.py b/octoeverywhere/octohttprequest.py index 8d645bc..a56bba5 100644 --- a/octoeverywhere/octohttprequest.py +++ b/octoeverywhere/octohttprequest.py @@ -1,4 +1,6 @@ import platform +import logging + import requests from .localip import LocalIpHelper @@ -138,7 +140,9 @@ def SetFullBodyBuffer(self, buffer, isZlibCompressed:bool = False, preCompressed # Since most things use request Stream=True, this is a helpful util that will read the entire # content of a request and return it. Note if the request has no defined length, this will read # as long as the stream will go. - def ReadAllContentFromStreamResponse(self) -> None: + # This function will not throw on failures, it will read as much as it can and then set the buffer. + # On a complete failure, the buffer will be set to None, so that should be checked. + def ReadAllContentFromStreamResponse(self, logger:logging.Logger) -> None: # Ensure we have a stream to read. if self._requestLibResponseObj is None: raise Exception("ReadAllContentFromStreamResponse was called on a result with no request lib Response object.") @@ -146,11 +150,15 @@ def ReadAllContentFromStreamResponse(self) -> None: # We can't simply use response.content, since streaming was enabled. # We need to use iter_content, since it will keep returning data until all is read. # We use a high chunk count, so most of the time it will read all of the content in one go. - for chunk in self._requestLibResponseObj.iter_content(10000000): - if buffer is None: - buffer = chunk - else: - buffer += chunk + try: + for chunk in self._requestLibResponseObj.iter_content(10000000): + if buffer is None: + buffer = chunk + else: + buffer += chunk + except Exception as e: + lengthStr = "[buffer is None]" if buffer is None else str(len(buffer)) + logger.warn(f"ReadAllContentFromStreamResponse got an exception. We will return the current buffer length of {lengthStr}, exception: {e}") self.SetFullBodyBuffer(buffer) @property diff --git a/octoeverywhere/octosessionimpl.py b/octoeverywhere/octosessionimpl.py index 065ff1c..4fbf22c 100644 --- a/octoeverywhere/octosessionimpl.py +++ b/octoeverywhere/octosessionimpl.py @@ -2,6 +2,7 @@ import struct import threading import traceback +import logging # @@ -15,6 +16,7 @@ from .serverauth import ServerAuthHelper from .sentry import Sentry from .ostypeidentifier import OsTypeIdentifier +from .threaddebug import ThreadDebug from .Proto import OctoStreamMessage from .Proto import HandshakeAck @@ -26,7 +28,7 @@ class OctoSession: - def __init__(self, octoStream, logger, printerId, privateKey, isPrimarySession, sessionId, uiPopupInvoker, pluginVersion, serverHostType, isCompanion): + def __init__(self, octoStream, logger:logging.Logger, printerId:str, privateKey:str, isPrimarySession:bool, sessionId, uiPopupInvoker, pluginVersion, serverHostType, isCompanion): self.ActiveWebStreams = {} self.ActiveWebStreamsLock = threading.Lock() self.IsAcceptingStreams = True @@ -297,6 +299,8 @@ def HandleMessage(self, msgBytes): except Exception as e: # If anything throws, we consider it a protocol failure. traceback.print_exc() + # We have seen "failed to create thread" here before, so we do this to debug that. + ThreadDebug.DoThreadDumpLogout(self.Logger) Sentry.Exception("Failed to handle octo message.", e) self.OnSessionError(0) return diff --git a/octoeverywhere/webcamhelper.py b/octoeverywhere/webcamhelper.py index 9121275..a5a9a2a 100644 --- a/octoeverywhere/webcamhelper.py +++ b/octoeverywhere/webcamhelper.py @@ -18,6 +18,7 @@ class WebcamSettingItem: # flipHBool & flipVBool & rotationInt must exist. # rotationInt must be 0, 90, 180, or 270 def __init__(self, name:str = "", snapshotUrl:str = "", streamUrl:str = "", flipHBool:bool = False, flipVBool:bool = False, rotationInt:int = 0): + self._name = "" self.Name = name self.SnapshotUrl = snapshotUrl self.StreamUrl = streamUrl @@ -25,6 +26,21 @@ def __init__(self, name:str = "", snapshotUrl:str = "", streamUrl:str = "", flip self.FlipV = flipVBool self.Rotation = rotationInt + + @property + def Name(self): + return self._name + + + @Name.setter + def Name(self, value): + # When the name is set, make sure we convert it to the string style we use internally. + # This ensures that the name can be used and is consistent across the platform. + if value is not None and len(value) > 0: + value = WebcamHelper.MoonrakerToInternalWebcamNameConvert(value) + self._name = value + + def Validate(self, logger:logging.Logger) -> bool: if self.Name is None or len(self.Name) == 0: logger.error(f"Name value in WebcamSettingItem is None or empty. {self.StreamUrl}") @@ -51,6 +67,14 @@ def Validate(self, logger:logging.Logger) -> bool: # setup. This includes USB based cameras, external IP based cameras, and OctoPrint instances that don't have a snapshot URL defined. class WebcamHelper: + # If no other index is specified, 0 is the default webcam index. + # This assumption is also made in the service and website, so it can't change. + c_DefaultWebcamIndex = 0 + + # We need to cap this so they aren't crazy long. + # However, this COULD mess with teh default camera name logic, since it matches off names. + c_MaxWebcamNameLength = 20 + # A header we apply to all snapshot and webcam streams so the client can get the correct transforms the user has setup. c_OeWebcamTransformHeaderKey = "x-oe-webcam-transform" @@ -79,8 +103,8 @@ def __init__(self, logger:logging.Logger, webcamPlatformHelperInterface, pluginD # Returns the snapshot URL from the settings. # Can be None if there is no snapshot URL set in the settings! # This URL can be absolute or relative. - def GetSnapshotUrl(self, cameraName:str = None): - obj = self._GetWebcamSettingObj(cameraName) + def GetSnapshotUrl(self, cameraIndex:int = None): + obj = self._GetWebcamSettingObj(cameraIndex) if obj is None: return None return obj.SnapshotUrl @@ -89,32 +113,32 @@ def GetSnapshotUrl(self, cameraName:str = None): # Returns the mjpeg stream URL from the settings. # Can be None if there is no URL set in the settings! # This URL can be absolute or relative. - def GetWebcamStreamUrl(self, cameraName:str = None): - obj = self._GetWebcamSettingObj(cameraName) + def GetWebcamStreamUrl(self, cameraIndex:int = None): + obj = self._GetWebcamSettingObj(cameraIndex) if obj is None: return None return obj.StreamUrl # Returns if flip H is set in the settings. - def GetWebcamFlipH(self, cameraName:str = None): - obj = self._GetWebcamSettingObj(cameraName) + def GetWebcamFlipH(self, cameraIndex:int = None): + obj = self._GetWebcamSettingObj(cameraIndex) if obj is None: return None return obj.FlipH # Returns if flip V is set in the settings. - def GetWebcamFlipV(self, cameraName:str = None): - obj = self._GetWebcamSettingObj(cameraName) + def GetWebcamFlipV(self, cameraIndex:int = None): + obj = self._GetWebcamSettingObj(cameraIndex) if obj is None: return None return obj.FlipV # Returns if rotate 90 is set in the settings. - def GetWebcamRotation(self, cameraName:str = None): - obj = self._GetWebcamSettingObj(cameraName) + def GetWebcamRotation(self, cameraIndex:int = None): + obj = self._GetWebcamSettingObj(cameraIndex) if obj is None: return None return obj.Rotation @@ -135,20 +159,20 @@ def IsWebcamStreamOracleRequest(self, requestHeadersDict): return "oe-webcamstream" in requestHeadersDict # If the header is set to specify a camera name, this returns it. Otherwise None - def GetOracleRequestCameraName(self, requestHeadersDict): - if "oe-webcam-name" in requestHeadersDict: - return requestHeadersDict["oe-webcam-name"] + def GetOracleRequestCameraIndex(self, requestHeadersDict) -> int: + if "oe-webcam-index" in requestHeadersDict: + return int(requestHeadersDict["oe-webcam-index"]) return None # Called by the OctoWebStreamHelper when a Oracle snapshot or webcam stream request is detected. # It's important that this function returns a OctoHttpRequest that's very similar to what the default MakeHttpCall function # returns, to ensure the rest of the octostream http logic can handle the response. def MakeSnapshotOrWebcamStreamRequest(self, httpInitialContext, method, sendHeaders, uploadBuffer) -> OctoHttpRequest.Result: - cameraNameOpt = self.GetOracleRequestCameraName(sendHeaders) + cameraIndexOpt = self.GetOracleRequestCameraIndex(sendHeaders) if self.IsSnapshotOracleRequest(sendHeaders): - return self.GetSnapshot(cameraNameOpt) + return self.GetSnapshot(cameraIndexOpt) elif self.IsWebcamStreamOracleRequest(sendHeaders): - return self.GetWebcamStream(cameraNameOpt) + return self.GetWebcamStream(cameraIndexOpt) else: raise Exception("Webcam helper MakeSnapshotOrWebcamStreamRequest was called but the request didn't have the oracle headers?") @@ -158,18 +182,18 @@ def MakeSnapshotOrWebcamStreamRequest(self, httpInitialContext, method, sendHead # # On failure, this returns None. Returning None will fail out the request. # On success, this will return a valid OctoHttpRequest. - def GetWebcamStream(self, cameraName:str = None) -> OctoHttpRequest.Result: + def GetWebcamStream(self, cameraIndex:int = None) -> OctoHttpRequest.Result: # Wrap the entire result in the add transform function, so on success the header gets added. - return self._AddOeWebcamTransformHeader(self._GetWebcamStreamInternal(cameraName), cameraName) + return self._AddOeWebcamTransformHeader(self._GetWebcamStreamInternal(cameraIndex), cameraIndex) - def _GetWebcamStreamInternal(self, cameraName:str) -> OctoHttpRequest.Result: + def _GetWebcamStreamInternal(self, cameraIndex:int) -> OctoHttpRequest.Result: # Check if the platform helper has an override. If so, it is responsible for all of the stream getting logic. if hasattr(self.WebcamPlatformHelperInterface, 'GetStream_Override'): - return self.WebcamPlatformHelperInterface.GetStream_Override(cameraName) + return self.WebcamPlatformHelperInterface.GetStream_Override(cameraIndex) # Try to get the URL from the settings. - webcamStreamUrl = self.GetWebcamStreamUrl(cameraName) + webcamStreamUrl = self.GetWebcamStreamUrl(cameraIndex) if webcamStreamUrl is not None: # Try to make a standard http call with this stream url # Use use this HTTP call helper system because it might be somewhat tricky to know @@ -188,19 +212,19 @@ def _GetWebcamStreamInternal(self, cameraName:str) -> OctoHttpRequest.Result: # # On failure, this returns None. Returning None will fail out the request. # On success, this will return a valid OctoHttpRequest that's fully filled out. The stream will always already be fully read, and will be FullBodyBuffer var. - def GetSnapshot(self, cameraName:str = None) -> OctoHttpRequest.Result: + def GetSnapshot(self, cameraIndex:int = None) -> OctoHttpRequest.Result: # Wrap the entire result in the _EnsureJpegHeaderInfo function, so ensure the returned snapshot can be used by all image processing libs. # Wrap the entire result in the add transform function, so on success the header gets added. - return self._AddOeWebcamTransformHeader(self._EnsureJpegHeaderInfo(self._GetSnapshotInternal(cameraName)), cameraName) + return self._AddOeWebcamTransformHeader(self._EnsureJpegHeaderInfo(self._GetSnapshotInternal(cameraIndex)), cameraIndex) - def _GetSnapshotInternal(self, cameraName:str) -> OctoHttpRequest.Result: + def _GetSnapshotInternal(self, cameraIndex:int) -> OctoHttpRequest.Result: # Check if the platform helper has an override. If so, it is responsible for all of the snapshot getting logic. if hasattr(self.WebcamPlatformHelperInterface, 'GetSnapshot_Override'): - return self.WebcamPlatformHelperInterface.GetSnapshot_Override(cameraName) + return self.WebcamPlatformHelperInterface.GetSnapshot_Override(cameraIndex) # First, try to get the snapshot using the string defined in settings. - snapshotUrl = self.GetSnapshotUrl(cameraName) + snapshotUrl = self.GetSnapshotUrl(cameraIndex) if snapshotUrl is not None: # Try to make a standard http call with this snapshot url # Use use this HTTP call helper system because it might be somewhat tricky to know @@ -375,23 +399,24 @@ def _GetSnapshotFromStream(self, url) -> OctoHttpRequest.Result: # Returns the default webcam setting object or None if there isn't one. - def _GetWebcamSettingObj(self, cameraName:str = None): + # If there isn't a default webcam name, it's assumed to be the first webcam returned in the list command. + def _GetWebcamSettingObj(self, cameraIndex:int = None): try: - a = self.ListWebcams() - if a is None or len(a) == 0: + # Get the current list of webcam settings. + webcamItems = self.ListWebcams() + if webcamItems is None or len(webcamItems) == 0: return None - # If a camera name wasn't passed, see if there's a default. - if cameraName is None or len(cameraName) == 0: - cameraName = self.GetDefaultCameraName() - # If we have a target name, see if we can find it. - if cameraName is not None: - cameraNameLower = cameraName.lower() - for i in a: - if i.Name.lower() == cameraNameLower: - return i - self.Logger.warn(f"_GetWebcamSettingObj asked for {cameraName} but we didn't find it.") - # Otherwise, default to the first entry. - return a[0] + + # If a camera index wasn't passed, get the default index. + if cameraIndex is None: + cameraIndex = self.GetDefaultCameraIndex(webcamItems) + + # We will always get a default index back from the above function. + if cameraIndex is not None and cameraIndex >= 0 and cameraIndex < len(webcamItems): + return webcamItems[cameraIndex] + + self.Logger.warn(f"_GetWebcamSettingObj asked for {cameraIndex} but it was out of bounds. Max: {len(webcamItems)}") + return webcamItems[WebcamHelper.c_DefaultWebcamIndex] except Exception as e: Sentry.Exception("WebcamHelper _GetWebcamSettingObj exception.", e) return None @@ -413,7 +438,7 @@ def ListWebcams(self): # Checks if the result was success and if so adds the common header. # Returns the octoHttpResult, so the function is chainable - def _AddOeWebcamTransformHeader(self, octoHttpResult, cameraName:str): + def _AddOeWebcamTransformHeader(self, octoHttpResult, cameraIndex:int): if octoHttpResult is None: return octoHttpResult @@ -421,7 +446,7 @@ def _AddOeWebcamTransformHeader(self, octoHttpResult, cameraName:str): transformStr = "none" # If there are any settings build a string with them all contaminated. - settings = self._GetWebcamSettingObj(cameraName) + settings = self._GetWebcamSettingObj(cameraIndex) if settings.FlipH or settings.FlipV or settings.Rotation != 0: transformStr = "" if settings.FlipH: @@ -452,7 +477,7 @@ def _EnsureJpegHeaderInfo(self, octoHttpResult:OctoHttpRequest.Result): buf = octoHttpResult.FullBodyBuffer if buf is None: # This will read the entire stream and store it into the FullBodyBuffer - octoHttpResult.ReadAllContentFromStreamResponse() + octoHttpResult.ReadAllContentFromStreamResponse(self.Logger) buf = octoHttpResult.FullBodyBuffer if buf is None: self.Logger.error("_EnsureJpegHeaderInfo got a null body read from ReadAllContentFromStreamResponse") @@ -619,6 +644,8 @@ def FixMissingSlashInWebcamUrlIfNeeded(logger:logging.Logger, webcamUrl:str) -> # # Default camera name logic. + # The default camera is always set and stored as the name, since the camera index can change over time. + # But it's always gotten as the index of the current list of cameras. # # Sets the default camera name and writes it to the settings file. @@ -635,9 +662,23 @@ def SetDefaultCameraName(self, name:str) -> None: self.Logger.error("SetDefaultCameraName failed "+str(e)) - # Returns the default camera name or None - def GetDefaultCameraName(self) -> str: - return self.DefaultCameraName + # Returns the default camera index. This will always return an int. + # If there is not a default currently set, this returns the WebcamHelper.c_DefaultWebcamIndex, which is index 0. + def GetDefaultCameraIndex(self, webcamItemList) -> int: + # If there is no name currently, the default is 0. + if self.DefaultCameraName is None: + return WebcamHelper.c_DefaultWebcamIndex + + # Try to find the name that was last set. + defaultCameraNameLower = self.DefaultCameraName.lower() + count = 0 + for i in webcamItemList: + if i.Name == defaultCameraNameLower: + return count + count += 1 + + # We didn't find it, return the default. + return WebcamHelper.c_DefaultWebcamIndex # Loads the current name from our settings file. @@ -661,3 +702,14 @@ def _LoadDefaultCameraName(self) -> None: self.Logger.info(f"Webcam settings loaded. Default camera name: {self.DefaultCameraName}") except Exception as e: self.Logger.error("_LoadDefaultCameraName failed "+str(e)) + + + @staticmethod + def MoonrakerToInternalWebcamNameConvert(name:str): + if name is not None and len(name) > 0: + # Enforce max name length. + if len(name) > WebcamHelper.c_MaxWebcamNameLength: + name = name[WebcamHelper.c_MaxWebcamNameLength] + # Ensure the string is only utf8 + name = name.encode('utf-8', 'ignore').decode('utf-8') + return name diff --git a/octoeverywhere/websocketimpl.py b/octoeverywhere/websocketimpl.py index f919444..44b9f35 100644 --- a/octoeverywhere/websocketimpl.py +++ b/octoeverywhere/websocketimpl.py @@ -91,9 +91,13 @@ def fireWsErrorCallbackThread(self, exception): # ignore any exceptions. try: self.Ws.close() - except Exception as ex : - Sentry.Exception("Websocket fireWsErrorCallbackThread close exception", ex) - + except Exception as e: + # This is a known bug in the websocket class, it happens when the WS is closing. + if isinstance(e, AttributeError) and "object has no attribute 'close'" in str(e): + # We don't have a logger, sooooooo + print("Websocket closed due to: 'NoneType' object has no attribute 'close'") + else: + Sentry.Exception("Websocket fireWsErrorCallbackThread close exception", e) except Exception as e : Sentry.Exception("Websocket client exception in fireWsErrorCallbackThread", e) @@ -179,6 +183,12 @@ def IsCommonConnectionException(e:Exception): # or there was a DNS name resolve failure. if isinstance(e, websocket.WebSocketConnectionClosedException) and ("Connection to remote host was lost." in str(e) or "ping/pong timed out" in str(e) or "Name or service not known" in str(e)): return True + # Invalid host name. + if isinstance(e, websocket.WebSocketAddressException) and "Name or service not known" in str(e): + return True + # We don't care. + if isinstance(e. WebSocketConnectionClosedException): + return True except Exception: pass return False diff --git a/octoprint_octoeverywhere/__init__.py b/octoprint_octoeverywhere/__init__.py index 9cc78a7..df7567e 100644 --- a/octoprint_octoeverywhere/__init__.py +++ b/octoprint_octoeverywhere/__init__.py @@ -567,7 +567,11 @@ def CheckIfPrinterIsSetupAndShowMessageIfNot(self): self.ShowUiPopup(title, message, "notice", "Finish Your Setup Now", addPrinterUrl, 20, False) except Exception as e: - Sentry.Exception("CheckIfPrinterIsSetupAndShowMessageIfNot failed", e) + if "Temporary failure in name resolution" in str(e): + # Ignore this temp issue. + pass + else: + Sentry.Exception("CheckIfPrinterIsSetupAndShowMessageIfNot failed", e) # Ensures we have generated a printer id and returns it. diff --git a/octoprint_octoeverywhere/slipstream.py b/octoprint_octoeverywhere/slipstream.py index 4fb05bc..e3d9732 100644 --- a/octoprint_octoeverywhere/slipstream.py +++ b/octoprint_octoeverywhere/slipstream.py @@ -290,7 +290,7 @@ def _GetCacheReadyOctoHttpResult(self, url): # This is actually a good idea, so the request connection doesn't hang around for a long time. buffer = None try: - octoHttpResult.ReadAllContentFromStreamResponse() + octoHttpResult.ReadAllContentFromStreamResponse(self.Logger) buffer = octoHttpResult.FullBodyBuffer except Exception as e: self.Logger.error("Slipstream failed to read index buffer for "+url+", e:"+str(e)) diff --git a/setup.py b/setup.py index 4841b62..4df01d6 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this is also parsed by the moonraker module to pull the version, so the string and format must remain the same! -plugin_version = "3.0.8" +plugin_version = "3.1.0" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 5d45c17de725496f27392526289a503dd8a3f10e Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Mon, 18 Mar 2024 22:06:17 -0700 Subject: [PATCH 057/328] Minor webcam name fixes for OctoPrint. --- octoeverywhere/webcamhelper.py | 4 ++++ octoprint_octoeverywhere/octoprintwebcamhelper.py | 9 ++++++++- setup.py | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/octoeverywhere/webcamhelper.py b/octoeverywhere/webcamhelper.py index a5a9a2a..8576b5a 100644 --- a/octoeverywhere/webcamhelper.py +++ b/octoeverywhere/webcamhelper.py @@ -712,4 +712,8 @@ def MoonrakerToInternalWebcamNameConvert(name:str): name = name[WebcamHelper.c_MaxWebcamNameLength] # Ensure the string is only utf8 name = name.encode('utf-8', 'ignore').decode('utf-8') + # Make the first letter uppercase + name = name[0].upper() + name[1:] + # If there are any / they will break our UI, so remove them. + name = name.replace("/", "") return name diff --git a/octoprint_octoeverywhere/octoprintwebcamhelper.py b/octoprint_octoeverywhere/octoprintwebcamhelper.py index d768ca3..aa87141 100644 --- a/octoprint_octoeverywhere/octoprintwebcamhelper.py +++ b/octoprint_octoeverywhere/octoprintwebcamhelper.py @@ -78,6 +78,13 @@ def GetWebcamConfig(self): if webcam.canSnapshot is False: self.Logger.info(f"We found a webcam {webcamName} but it doesn't support snapshots, we will try to detect the snapshot URL for ourselves.") + # We found that some of the webcam plugins do fun things with the names, so we clean them up for the UI. + # The multicam plugin prefixes them with multicam/Camera Name + if webcamName is not None: + slashPos = webcamName.find("/") + if slashPos != -1: + webcamName = webcamName[slashPos+1:] + # Make an empty webcam settings item to fill. webSettingsItem = WebcamSettingItem(webcamName) @@ -120,7 +127,7 @@ def GetWebcamConfig(self): # Ensure we have everything required. if webSettingsItem.Validate(self.Logger): results.append(webSettingsItem) - self.Logger.debug(f"Webcam found. Name: {webcamName}, {webSettingsItem.StreamUrl}, {webSettingsItem.SnapshotUrl}, {webSettingsItem.FlipH}, {webSettingsItem.FlipV}, {webSettingsItem.Rotation}") + self.Logger.debug(f"Webcam found. Name: {webSettingsItem.Name}, {webSettingsItem.StreamUrl}, {webSettingsItem.SnapshotUrl}, {webSettingsItem.FlipH}, {webSettingsItem.FlipV}, {webSettingsItem.Rotation}") else: self.Logger.debug(f"Webcam settings item validation failed for {webcamName}") diff --git a/setup.py b/setup.py index 4df01d6..45d7b72 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this is also parsed by the moonraker module to pull the version, so the string and format must remain the same! -plugin_version = "3.1.0" +plugin_version = "3.1.1" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 705258a92cecc95dd457bcda93bc0d99ed232dca Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Fri, 22 Mar 2024 09:27:49 -0700 Subject: [PATCH 058/328] Adding the first pass of camera support for the X1 --- .vscode/launch.json | 39 +- .vscode/settings.json | 5 + bambu_octoeverywhere/bambuclient.py | 5 +- bambu_octoeverywhere/bambucommandhandler.py | 7 +- bambu_octoeverywhere/bambumodels.py | 32 +- bambu_octoeverywhere/quickcam.py | 434 ++++++++++++++++---- 6 files changed, 411 insertions(+), 111 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 7c6234d..2851e15 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -4,39 +4,54 @@ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ + // { + // "name": "Moonraker - PI - Single Instance", + // "type": "debugpy", + // "request": "launch", + // "module": "moonraker_octoeverywhere", + // "justMyCode": false, + // "args": [ + // // These args reflect the correct setup for a pi installed single instance of OctoEverywhere connecting to a local Moonraker. + // // The string is a urlBase64 encoded string of json. We base64 encode it to prevent any issues with command line args. + // // + // "eyJLbGlwcGVyQ29uZmlnRm9sZGVyIjogIi9ob21lL3BpL3ByaW50ZXJfZGF0YS9jb25maWciLCAiTW9vbnJha2VyQ29uZmlnRmlsZSI6ICIvaG9tZS9waS9wcmludGVyX2RhdGEvY29uZmlnL21vb25yYWtlci5jb25mIiwgIktsaXBwZXJMb2dGb2xkZXIiOiAiL2hvbWUvcGkvcHJpbnRlcl9kYXRhL2xvZ3MiLCAiTG9jYWxGaWxlU3RvcmFnZVBhdGgiOiAiL2hvbWUvcGkvcHJpbnRlcl9kYXRhL29jdG9ldmVyeXdoZXJlLXN0b3JlIiwgIlNlcnZpY2VOYW1lIjogIm9jdG9ldmVyeXdoZXJlIiwgIlZpcnR1YWxFbnZQYXRoIjogIi9ob21lL3BpL29jdG9ldmVyeXdoZXJlLWVudiIsICJSZXBvUm9vdEZvbGRlciI6ICIvaG9tZS9waS9vY3RvZXZlcnl3aGVyZSJ9", + // // + // // We can optionally pass a dev config json object, which has dev specific overwrites we can make. + // "{\"LocalServerAddress\":\"\", \"LogLevel\":\"DEBUG\"}" + // ] + // }, { - "name": "Moonraker - PI - Single Instance", + "name": "Moonraker - Klipper", "type": "debugpy", "request": "launch", "module": "moonraker_octoeverywhere", "justMyCode": false, "args": [ - // These args reflect the correct setup for a pi installed single instance of OctoEverywhere connecting to a local Moonraker. + // These args reflect the correct setup for a pi installed multiple instances of OctoEverywhere connecting to a local Moonraker. These args target the first instance. // The string is a urlBase64 encoded string of json. We base64 encode it to prevent any issues with command line args. // - "eyJLbGlwcGVyQ29uZmlnRm9sZGVyIjogIi9ob21lL3BpL3ByaW50ZXJfZGF0YS9jb25maWciLCAiTW9vbnJha2VyQ29uZmlnRmlsZSI6ICIvaG9tZS9waS9wcmludGVyX2RhdGEvY29uZmlnL21vb25yYWtlci5jb25mIiwgIktsaXBwZXJMb2dGb2xkZXIiOiAiL2hvbWUvcGkvcHJpbnRlcl9kYXRhL2xvZ3MiLCAiTG9jYWxGaWxlU3RvcmFnZVBhdGgiOiAiL2hvbWUvcGkvcHJpbnRlcl9kYXRhL29jdG9ldmVyeXdoZXJlLXN0b3JlIiwgIlNlcnZpY2VOYW1lIjogIm9jdG9ldmVyeXdoZXJlIiwgIlZpcnR1YWxFbnZQYXRoIjogIi9ob21lL3BpL29jdG9ldmVyeXdoZXJlLWVudiIsICJSZXBvUm9vdEZvbGRlciI6ICIvaG9tZS9waS9vY3RvZXZlcnl3aGVyZSJ9", - // + "eyJLbGlwcGVyQ29uZmlnRm9sZGVyIjogIi9ob21lL3BpL3ByaW50ZXJfMV9kYXRhL2NvbmZpZyIsICJNb29ucmFrZXJDb25maWdGaWxlIjogIi9ob21lL3BpL3ByaW50ZXJfMV9kYXRhL2NvbmZpZy9tb29ucmFrZXIuY29uZiIsICJLbGlwcGVyTG9nRm9sZGVyIjogIi9ob21lL3BpL3ByaW50ZXJfMV9kYXRhL2xvZ3MiLCAiTG9jYWxGaWxlU3RvcmFnZVBhdGgiOiAiL2hvbWUvcGkvcHJpbnRlcl8xX2RhdGEvb2N0b2V2ZXJ5d2hlcmUtc3RvcmUiLCAiU2VydmljZU5hbWUiOiAib2N0b2V2ZXJ5d2hlcmUtMSIsICJWaXJ0dWFsRW52UGF0aCI6ICIvaG9tZS9waS9vY3RvZXZlcnl3aGVyZS1lbnYiLCAiUmVwb1Jvb3RGb2xkZXIiOiAiL2hvbWUvcGkvb2N0b2V2ZXJ5d2hlcmUifQ==", // We can optionally pass a dev config json object, which has dev specific overwrites we can make. "{\"LocalServerAddress\":\"\", \"LogLevel\":\"DEBUG\"}" ] }, { - "name": "Moonraker - PI - Multi Instance", + "name": "Bambu Connect - P1S", "type": "debugpy", "request": "launch", - "module": "moonraker_octoeverywhere", + "module": "bambu_octoeverywhere", "justMyCode": false, "args": [ - // These args reflect the correct setup for a pi installed multiple instances of OctoEverywhere connecting to a local Moonraker. These args target the first instance. - // The string is a urlBase64 encoded string of json. We base64 encode it to prevent any issues with command line args. + // These args reflect the correct setup for a pi installed with the Bambu Connect version of the plugin. These args target the first instance. // - "eyJLbGlwcGVyQ29uZmlnRm9sZGVyIjogIi9ob21lL3BpL3ByaW50ZXJfMV9kYXRhL2NvbmZpZyIsICJNb29ucmFrZXJDb25maWdGaWxlIjogIi9ob21lL3BpL3ByaW50ZXJfMV9kYXRhL2NvbmZpZy9tb29ucmFrZXIuY29uZiIsICJLbGlwcGVyTG9nRm9sZGVyIjogIi9ob21lL3BpL3ByaW50ZXJfMV9kYXRhL2xvZ3MiLCAiTG9jYWxGaWxlU3RvcmFnZVBhdGgiOiAiL2hvbWUvcGkvcHJpbnRlcl8xX2RhdGEvb2N0b2V2ZXJ5d2hlcmUtc3RvcmUiLCAiU2VydmljZU5hbWUiOiAib2N0b2V2ZXJ5d2hlcmUtMSIsICJWaXJ0dWFsRW52UGF0aCI6ICIvaG9tZS9waS9vY3RvZXZlcnl3aGVyZS1lbnYiLCAiUmVwb1Jvb3RGb2xkZXIiOiAiL2hvbWUvcGkvb2N0b2V2ZXJ5d2hlcmUifQ==", + // { "ServiceName": "octoeverywhere-bambu", "VirtualEnvPath":"/home/pi/octoeverywhere-env", "RepoRootFolder":"/home/pi/octoeverywhere/", "LogFolder":"/home/pi/.octoeverywhere-bambu/logs", "LocalFileStoragePath":"/home/pi/.octoeverywhere-bambu/octoeverywhere-store", "ConfigFolder":"/home/pi/.octoeverywhere-bambu/", "CompanionInstanceIdStr":"1" } + "eyAiU2VydmljZU5hbWUiOiAib2N0b2V2ZXJ5d2hlcmUtYmFtYnUiLCAiVmlydHVhbEVudlBhdGgiOiIvaG9tZS9waS9vY3RvZXZlcnl3aGVyZS1lbnYiLCAiUmVwb1Jvb3RGb2xkZXIiOiIvaG9tZS9waS9vY3RvZXZlcnl3aGVyZS8iLCAiTG9nRm9sZGVyIjoiL2hvbWUvcGkvLm9jdG9ldmVyeXdoZXJlLWJhbWJ1L2xvZ3MiLCAiTG9jYWxGaWxlU3RvcmFnZVBhdGgiOiIvaG9tZS9waS8ub2N0b2V2ZXJ5d2hlcmUtYmFtYnUvb2N0b2V2ZXJ5d2hlcmUtc3RvcmUiLCAiQ29uZmlnRm9sZGVyIjoiL2hvbWUvcGkvLm9jdG9ldmVyeXdoZXJlLWJhbWJ1LyIsICJDb21wYW5pb25JbnN0YW5jZUlkU3RyIjoiMSIgfQ==", // We can optionally pass a dev config json object, which has dev specific overwrites we can make. "{\"LocalServerAddress\":\"\", \"LogLevel\":\"DEBUG\"}" ] }, { - "name": "Bambu Connect - PI - Instnace 1", + "name": "Bambu Connect - X1C", "type": "debugpy", "request": "launch", "module": "bambu_octoeverywhere", @@ -44,8 +59,8 @@ "args": [ // These args reflect the correct setup for a pi installed with the Bambu Connect version of the plugin. These args target the first instance. // - // { "ServiceName": "octoeverywhere-bambu", "VirtualEnvPath":"/home/pi/octoeverywhere-env", "RepoRootFolder":"/home/pi/octoeverywhere/", "LogFolder":"/home/pi/.octoeverywhere-bambu/logs", "LocalFileStoragePath":"/home/pi/.octoeverywhere-bambu/octoeverywhere-store", "ConfigFolder":"/home/pi/.octoeverywhere-bambu/", "CompanionInstanceIdStr":"1" } - "eyAiU2VydmljZU5hbWUiOiAib2N0b2V2ZXJ5d2hlcmUtYmFtYnUiLCAiVmlydHVhbEVudlBhdGgiOiIvaG9tZS9waS9vY3RvZXZlcnl3aGVyZS1lbnYiLCAiUmVwb1Jvb3RGb2xkZXIiOiIvaG9tZS9waS9vY3RvZXZlcnl3aGVyZS8iLCAiTG9nRm9sZGVyIjoiL2hvbWUvcGkvLm9jdG9ldmVyeXdoZXJlLWJhbWJ1L2xvZ3MiLCAiTG9jYWxGaWxlU3RvcmFnZVBhdGgiOiIvaG9tZS9waS8ub2N0b2V2ZXJ5d2hlcmUtYmFtYnUvb2N0b2V2ZXJ5d2hlcmUtc3RvcmUiLCAiQ29uZmlnRm9sZGVyIjoiL2hvbWUvcGkvLm9jdG9ldmVyeXdoZXJlLWJhbWJ1LyIsICJDb21wYW5pb25JbnN0YW5jZUlkU3RyIjoiMSIgfQ==", + // { "ServiceName": "octoeverywhere-bambu-2", "VirtualEnvPath":"/home/pi/octoeverywhere-env", "RepoRootFolder":"/home/pi/octoeverywhere/", "LogFolder":"/home/pi/.octoeverywhere-bambu-2/logs", "LocalFileStoragePath":"/home/pi/.octoeverywhere-bambu-2/octoeverywhere-store", "ConfigFolder":"/home/pi/.octoeverywhere-bambu-2/", "CompanionInstanceIdStr":"1" } + "eyAiU2VydmljZU5hbWUiOiAib2N0b2V2ZXJ5d2hlcmUtYmFtYnUtMiIsICJWaXJ0dWFsRW52UGF0aCI6Ii9ob21lL3BpL29jdG9ldmVyeXdoZXJlLWVudiIsICJSZXBvUm9vdEZvbGRlciI6Ii9ob21lL3BpL29jdG9ldmVyeXdoZXJlLyIsICJMb2dGb2xkZXIiOiIvaG9tZS9waS8ub2N0b2V2ZXJ5d2hlcmUtYmFtYnUtMi9sb2dzIiwgIkxvY2FsRmlsZVN0b3JhZ2VQYXRoIjoiL2hvbWUvcGkvLm9jdG9ldmVyeXdoZXJlLWJhbWJ1LTIvb2N0b2V2ZXJ5d2hlcmUtc3RvcmUiLCAiQ29uZmlnRm9sZGVyIjoiL2hvbWUvcGkvLm9jdG9ldmVyeXdoZXJlLWJhbWJ1LTIvIiwgIkNvbXBhbmlvbkluc3RhbmNlSWRTdHIiOiIxIiB9", // We can optionally pass a dev config json object, which has dev specific overwrites we can make. "{\"LocalServerAddress\":\"\", \"LogLevel\":\"DEBUG\"}" ] diff --git a/.vscode/settings.json b/.vscode/settings.json index 95f4b62..676450d 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -46,6 +46,7 @@ "didnt", "dnspython", "esac", + "faststart", "filamentchange", "filemetadatacache", "finalsnap", @@ -103,6 +104,7 @@ "moonrakerdatabase", "moonrakerhost", "moonrakerwebcamhelper", + "movflags", "mqtt", "msgcount", "multicam", @@ -175,6 +177,8 @@ "reqs", "requestsutils", "routable", + "RTSP", + "rtsps", "sdcard", "serverauth", "shotty", @@ -213,6 +217,7 @@ "VIEWMODELS", "virt", "virtualenv", + "wallclock", "warmingup", "webassets", "webcamhelper", diff --git a/bambu_octoeverywhere/bambuclient.py b/bambu_octoeverywhere/bambuclient.py index 331a053..23fe3b1 100644 --- a/bambu_octoeverywhere/bambuclient.py +++ b/bambu_octoeverywhere/bambuclient.py @@ -20,6 +20,9 @@ class BambuClient: _Instance = None + # Useful for debugging. + _PrintMQTTMessages = False + @staticmethod def Init(logger:logging.Logger, config:Config, stateTranslator): BambuClient._Instance = BambuClient(logger, config, stateTranslator) @@ -234,7 +237,7 @@ def _OnMessage(self, client, userdata, mqttMsg:mqtt.MQTTMessage): raise Exception("Parsed json MQTT message returned None") # Print for debugging if desired. - if self.Logger.isEnabledFor(logging.DEBUG): + if BambuClient._PrintMQTTMessages and self.Logger.isEnabledFor(logging.DEBUG): self.Logger.debug("Incoming Bambu Message:\r\n"+json.dumps(msg, indent=3)) # Since we keep a track of the state locally from the partial updates, we need to feed all updates to our state object. diff --git a/bambu_octoeverywhere/bambucommandhandler.py b/bambu_octoeverywhere/bambucommandhandler.py index 22c0c60..94e9c0d 100644 --- a/bambu_octoeverywhere/bambucommandhandler.py +++ b/bambu_octoeverywhere/bambucommandhandler.py @@ -66,7 +66,12 @@ def GetCurrentJobStatus(self): elif gcodeState == "PAUSE": state = "paused" elif gcodeState == "FINISH": - state = "complete" + # When the X1C first starts and does the first time user calibration, the state is FINISH + # but there's really nothing done. This might happen after other calibrations, so if the total layers is 0, we are idle. + if bambuState.total_layer_num is not None and bambuState.total_layer_num == 0: + state = "idle" + else: + state = "complete" elif gcodeState == "FAILED": state = "cancelled" elif gcodeState == "PREPARE": diff --git a/bambu_octoeverywhere/bambumodels.py b/bambu_octoeverywhere/bambumodels.py index e222a13..9d4bfdb 100644 --- a/bambu_octoeverywhere/bambumodels.py +++ b/bambu_octoeverywhere/bambumodels.py @@ -2,6 +2,7 @@ import logging from enum import Enum +from octoeverywhere.sentry import Sentry # Known printer error types. # Note that the print state doesn't have to be ERROR to have an error, during a print it's "PAUSED" but the print_error value is not 0. @@ -22,7 +23,7 @@ def __init__(self) -> None: self.gcode_state:str = None self.layer_num:int = None self.total_layer_num:int = None - self.gcode_file:str = None + self.subtask_name:str = None self.mc_percent:int = None self.nozzle_temper:int = None self.nozzle_target_temper:int = None @@ -31,6 +32,10 @@ def __init__(self) -> None: self.mc_remaining_time:int = None self.project_id:str = None self.print_error:int = None + # On the X1, this is empty is LAN viewing of off + # It's a URL if streaming is enabled + # On other printers, this doesn't exist, so it's None + self.rtsp_url:str = None # Custom fields self.LastTimeRemainingWallClock:float = None @@ -43,7 +48,7 @@ def OnUpdate(self, msg:dict) -> None: self.gcode_state = msg.get("gcode_state", self.gcode_state) self.layer_num = msg.get("layer_num", self.layer_num) self.total_layer_num = msg.get("total_layer_num", self.total_layer_num) - self.gcode_file = msg.get("gcode_file", self.gcode_file) + self.subtask_name = msg.get("subtask_name", self.subtask_name) self.project_id = msg.get("project_id", self.project_id) self.mc_percent = msg.get("mc_percent", self.mc_percent) self.nozzle_temper = msg.get("nozzle_temper", self.nozzle_temper) @@ -51,6 +56,9 @@ def OnUpdate(self, msg:dict) -> None: self.bed_temper = msg.get("bed_temper", self.bed_temper) self.bed_target_temper = msg.get("bed_target_temper", self.bed_target_temper) self.print_error = msg.get("print_error", self.print_error) + ipCam = msg.get("ipcam", None) + if ipCam is not None: + self.rtsp_url = ipCam.get("rtsp_url", self.rtsp_url) # Time remaining has some custom logic, so as it's queried each time it keep counting down in seconds, since Bambu only gives us minutes. old_mc_remaining_time = self.mc_remaining_time @@ -92,12 +100,12 @@ def IsPaused(self) -> bool: # If there is a file name, this returns it without the final . def GetFileNameWithNoExtension(self): - if self.gcode_file is None: + if self.subtask_name is None: return None - pos = self.gcode_file.rfind(".") + pos = self.subtask_name.rfind(".") if pos == -1: - return self.gcode_file - return self.gcode_file[:pos] + return self.subtask_name + return self.subtask_name[:pos] # Returns a unique string for this print. @@ -123,6 +131,11 @@ def GetPrinterError(self) -> BambuPrintErrors: if self.print_error == 83918896 or self.print_error == 50364434 or self.print_error == 83935249: return None + # This state is when the user is loading filament, and the printer is asking them to push it in. + # This isn't an error. + if self.print_error == 134184967: + return None + # There's a full list of errors here, we only care about some of them # https://e.bambulab.com/query.php?lang=en # We format the error into a hex the same way the are on the page, to make it easier. @@ -217,7 +230,12 @@ def OnUpdate(self, msg:dict) -> None: self.PrinterName = BambuPrinters.A1 if self.PrinterName is None or self.PrinterName is BambuPrinters.Unknown: - self.Logger.warn(f"Unknown printer type. CPU:{self.Cpu}, Project Name: {self.ProjectName}, Hardware Version: {self.HardwareVersion}") + Sentry.LogError(f"Unknown printer type. CPU:{self.Cpu}, Project Name: {self.ProjectName}, Hardware Version: {self.HardwareVersion}",{ + "CPU": str(self.Cpu), + "ProjectName": str(self.ProjectName), + "HardwareVersion": str(self.HardwareVersion), + "SoftwareVersion": str(self.SoftwareVersion), + }) self.PrinterName = BambuPrinters.Unknown if self.HasLoggedPrinterVersion is False: diff --git a/bambu_octoeverywhere/quickcam.py b/bambu_octoeverywhere/quickcam.py index 6aee57c..5f1527d 100644 --- a/bambu_octoeverywhere/quickcam.py +++ b/bambu_octoeverywhere/quickcam.py @@ -1,9 +1,12 @@ import logging import threading +import subprocess +import selectors import struct import ssl import socket import time +import os from octoeverywhere.sentry import Sentry @@ -114,101 +117,79 @@ def _ensureCaptureThreadRunning(self): # Does the image image capture work. def _captureThread(self): try: - authData = bytearray() + # Get the access code and the host name. accessCode = self.Config.GetStr(Config.SectionBambu, Config.BambuAccessToken, None) ipOrHostname = self.Config.GetStr(Config.SectionCompanion, Config.CompanionKeyIpOrHostname, None) if accessCode is None or ipOrHostname is None: raise Exception("QuickCam doesn't have a access code or ip to use.") - # Build the auth packet - authData += struct.pack(" QuickCam.c_CaptureThreadTimeoutSec: - # TODO - For now, we don't stop the webcam loop while the printer is printing. - # This allows for notifications, Gadget, snapshots, streams, and such to load super easily. - # We need to measure the load on this though. - state = BambuClient.Get().GetState() - if state is None or not state.IsPrinting(True): - # This will invoke the finally clause and leave. - return - - # If the expected image size is 0, then this is the first read of 16 bytes for the header. - if expectedImageSize == 0: - if len(data) != 16: - raise Exception("QuickCam capture thread got a first payload that was longer than 16.") - expectedImageSize = int.from_bytes(data[0:3], byteorder='little') - # Otherwise, we are building an image - else: - # Always add to the current buffer. - imgBuffer += data - - # Check if the image is done. - if len(imgBuffer) == expectedImageSize: - # We have the full image. Sanity check the jpeg start and end bytes exist. - if imgBuffer[:4] != jpegStartSequence: - raise Exception("QuickCam got an image of the expected size, but we failed to find the jpeg start sequence.") - elif imgBuffer[-2:] != jpegEndSequence: - raise Exception("QuickCam got an image of the expected size, but we failed to find the jpeg end sequence.") - self._SetNewImage(imgBuffer) - expectedImageSize = 0 - imgBuffer = bytearray() - - # Sanity check we didn't get misaligned from the stream. - elif len(imgBuffer) > expectedImageSize: - raise Exception(f"QuickCam was building an image expected to be {expectedImageSize} but ended up with a buffer that was {imgBuffer}") - - except Exception as e: - # We have seen times where random errors are returned, like on boot or if the stream is opened too soon after closing. - # This exception block is designed to eat any connection or buffer parsing errors, eat them, and try again. - self.Logger.warn("Exception in QuickCam capture thread. "+str(e)) - time.sleep(2) + # TODO - Right now it seems the X1 doesn't send back version info on start or with the version command + # So we use the existence of the RTSP URL to determine what we can do. + # Ideally we would use the printer version in the future. + rtspUrl = None + verAttempt = 0 + while True: + verAttempt += 1 + state = BambuClient.Get().GetState() + # Wait until the object exists + if state is not None: + rtspUrl = state.rtsp_url + break + # If we can't get it return, and then the quick cam thread will be started again + # When there's another request. + if verAttempt > 5: + self.Logger.warn(f"QuickCam wasn't able to get the printer state after {verAttempt} attempts") + return + # Sleep for a bit. + time.sleep(2.0) + + # Create the camera implementation we need for this device. + camImpl = None + # Since we have to use the URL.... + # IF the URL is empty, it's an X1 with LAN streaming disabled. + # If the URL has an address, it's an X1 with LAN streaming. + # If it's None, it's a P1, A1, or another printer with no RTSP. + if rtspUrl is not None: + camImpl = QuickCam_RTSP(self.Logger) + else: + # Default to the websocket impl, since it's used on the most printers. + camImpl = QuickCam_WebSocket(self.Logger) + + # Wrap the usage into a with, so the connection is always cleaned up + with camImpl: + # We allow a few attempts, so if there are any connection issues or errors we buffer them out. + attempts = 0 + while attempts < 5: + attempts += 1 + try: + # Connect to the server. + camImpl.Connect(ipOrHostname, accessCode) + + # Begin the capture loop. + while True: + # Get the next image buffer. + # This can return None, which means we should just check the time and spin. + img = camImpl.GetImage() + + # Check if we are done running, if so, leave + if time.time() - self.LastImageRequestTimeSec > QuickCam.c_CaptureThreadTimeoutSec: + # TODO - For now, we don't stop the webcam loop while the printer is printing. + # This allows for notifications, Gadget, snapshots, streams, and such to load super easily. + # We need to measure the load on this though. + state = BambuClient.Get().GetState() + if state is None or not state.IsPrinting(True): + # This will invoke the finally clause and leave. + return + + # Set the image if we got one. + if img is not None: + self._SetNewImage(img) + + except Exception as e: + # We have seen times where random errors are returned, like on boot or if the stream is opened too soon after closing. + # This exception block is designed to eat any connection or buffer parsing errors, eat them, and try again. + self.Logger.warn("Exception in QuickCam capture thread. "+str(e)) + time.sleep(2) except Exception as e: Sentry.Exception("Exception in QuickCam capture thread. ", e) finally: @@ -222,3 +203,276 @@ def _captureThread(self): # And ensure that the current image is cleaned up, so clients don't get a stale image. self.CurrentImage = None self.Logger.info("QuickCam capture thread exit.") + + +# Implements the websocket camera version for the P1 and A1 series printers. +class QuickCam_WebSocket: + + def __init__(self, logger:logging.Logger): + self.Logger = logger + self.Socket = None + self.SslSocket = None + + # Image getting stuff + self.ImageBuffer = bytearray() + self.ExpectedImageSize = 0 + self.JpegStartSequence = bytearray([0xff, 0xd8, 0xff, 0xe0]) + self.JpegEndSequence = bytearray([0xff, 0xd9]) + + + # ~~ Interface Function ~~ + # Connects to the server. + # This will throw an exception if it fails. + def Connect(self, ipOrHostname:str, accessCode:str) -> None: + # Build the auth packet + authData = bytearray() + authData += struct.pack(" bytearray: + # Read from the socket + while True: + # We have seen this receive fail with SSLWantReadError when the socket if valid and there's more to read. In that case, keep the current socket going and try again. + data = None + try: + # We will read either the 16 byte header that starts every image or we will read the remainder of the current image. + readSize = 16 if self.ExpectedImageSize == 0 else self.ExpectedImageSize - len(self.ImageBuffer) + data = self.SslSocket.recv(readSize) + except ssl.SSLWantReadError: + time.sleep(1) + continue + + # If the expected image size is 0, then this is the first read of 16 bytes for the header. + if self.ExpectedImageSize == 0: + if len(data) != 16: + raise Exception("QuickCam capture thread got a first payload that was longer than 16.") + self.ExpectedImageSize = int.from_bytes(data[0:3], byteorder='little') + # Otherwise, we are building an image + else: + # Always add to the current buffer. + self.ImageBuffer += data + + # Check if the image is done. + if len(self.ImageBuffer) == self.ExpectedImageSize: + # We have the full image. Sanity check the jpeg start and end bytes exist. + if self.ImageBuffer[:4] != self.JpegStartSequence: + raise Exception("QuickCam got an image of the expected size, but we failed to find the jpeg start sequence.") + elif self.ImageBuffer[-2:] != self.JpegEndSequence: + raise Exception("QuickCam got an image of the expected size, but we failed to find the jpeg end sequence.") + self.ExpectedImageSize = 0 + temp = self.ImageBuffer + self.ImageBuffer = bytearray() + return temp + # Sanity check we didn't get misaligned from the stream. + elif len(self.ImageBuffer) > self.ExpectedImageSize: + raise Exception(f"QuickCam was building an image expected to be {self.ExpectedImageSize} but ended up with a buffer that was {self.ImageBuffer}") + + + # Allows us to using the with: scope. + def __enter__(self): + return self + + + # Allows us to using the with: scope. + # Must not throw! + def __exit__(self, t, v, tb): + # Close in the opposite order they were opened. + try: + if self.SslSocket is not None: + self.SslSocket.__exit__(t, v, tb) + except Exception: + pass + try: + if self.Socket is not None: + self.Socket.__exit__(t, v, tb) + except Exception: + pass + + +# Implements the websocket camera version for the X1 series printers. +class QuickCam_RTSP: + + def __init__(self, logger:logging.Logger): + self.Logger = logger + self.Process:subprocess.Popen = None + + # Image getting stuff + self.Buffer = bytearray() + self.SearchedIndex = 0 + self.JpegStartSequence = bytearray([0xff, 0xd8, 0xff, 0xfe, 0x00, 0x10]) + self.JpegStartSequenceLen = len(self.JpegStartSequence) + self.JpegEndSequence = bytearray([0xff, 0xd9]) + self.PipeSelect = selectors.DefaultSelector() + + + # ~~ Interface Function ~~ + # Connects to the server. + # This will throw an exception if it fails. + def Connect(self, ipOrHostname:str, accessCode:str) -> None: + # TODO check for ffmpeg + # TODO detect if ffmpeg has died or failed to run + # TODO get the address from the bambu state object + # Notes + # We use 15 fps because it's a good trade off of fps and cpu perf hits + # It also decreases the bandwidth needed, which helps on mobile + # We use the default jpeg image quality, for the same reasons above. + # pylint: disable=consider-using-with # We handle this on our own. + self.Process = subprocess.Popen(["ffmpeg", + "-hide_banner", + "-y", + "-loglevel", "error", + "-rtsp_transport", "tcp", + "-use_wallclock_as_timestamps", "1", + "-i", f"rtsps://bblp:{accessCode}@{ipOrHostname}:322/streaming/live/1", + "-filter:v", "fps=15", + "-movflags", "+faststart", + "-f", "image2pipe", "-" + ], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # pylint: disable=no-member # Linux only + os.set_blocking(self.Process.stdout.fileno(), False) + self.PipeSelect.register(self.Process.stdout, selectors.EVENT_READ) + + + # ~~ Interface Function ~~ + # Gets an image from the server. This should block until an image is ready. + # This can return None to indicate there's no image but the connection is still good, this allows the host to check if we should still be running. + # To indicate connection is closed or needs to be closed, this should throw. + def GetImage(self) -> bytearray: + while True: + # Wait on the pipe, which will signal us when there's data to be read. + self.PipeSelect.select() + + # Read all of the data we can. + buffer = self.Process.stdout.read(100000000) + + # If we get an empty buffer, we just need to wait for more. + if buffer is None or len(buffer) == 0: + continue + + # If there's no pending buffered data do a quick exit if we were able to read the entire buffer in our first read. + # If the full buffer is only one jpeg images, we don't need to do any scanning and can just return it. + if self.Buffer is None: + if self._CheckIfFullJpeg(buffer): + #self.Logger.info(f"quick image {len(buffer)}") + self._ResetLocalBuffer() + return buffer + + # Append this buffer to the current pending buffer. + if self.Buffer is None: + self.Buffer = buffer + else: + self.Buffer += buffer + + # Ensure the buffer is long enough to check. + buffLen = len(self.Buffer) + if buffLen <= self.JpegStartSequenceLen: + continue + + # Check if the buffer is a full image now + if self._CheckIfFullJpeg(self.Buffer): + img = self.Buffer + self._ResetLocalBuffer() + #self.Logger.info("second quick exit") + return img + + # Scan the buffer for the jpeg end sequence. + newImageStart = -1 + while self.SearchedIndex < buffLen - self.JpegStartSequenceLen: + if self.Buffer[self.SearchedIndex] == self.JpegEndSequence[0] and self.Buffer[self.SearchedIndex+1] == self.JpegEndSequence[1]: + newImageStart = self.SearchedIndex + 2 + break + self.SearchedIndex += 1 + + # See if we found a complete image. + if newImageStart != -1: + # Get the image and check it's a full image. + imgBuffer = self.Buffer[:newImageStart] + if self._CheckIfFullJpeg(imgBuffer) is False: + # If we don't have a correct buffer, we got off in out counting. + # So reset the buffer and continue. Note after we reset the buffer, we might + # hit this, since we could have a partial image in the Buffer + self._ResetLocalBuffer() + continue + # Take the image off the buffer. + self.Buffer = self.Buffer[newImageStart:] + self.SearchedIndex = 0 + # Ensure the buffer isn't too long. + self._ResetLocalBufferIfOverLimit() + return imgBuffer + + # If we didn't find anything, check the limit. + self._ResetLocalBufferIfOverLimit() + + + def _ResetLocalBufferIfOverLimit(self): + # A normal image is around 37,000, so if the buffer is too long, reset it so + # we can try to recover the buffer. + if self.Buffer is not None and len(self.Buffer) > 50000: + self.Logger.info("Quick cam rtsp buffer reset. This means we are running behind.") + self._ResetLocalBuffer() + + + def _ResetLocalBuffer(self): + self.SearchedIndex = 0 + self.Buffer = None + + + # Checks if the buffer is only one image, from start to end. + def _CheckIfFullJpeg(self, buffer:bytearray) -> bool: + if buffer is None or len(buffer) <= self.JpegStartSequenceLen: + return False + if buffer[:self.JpegStartSequenceLen] != self.JpegStartSequence: + return False + if buffer[-2:] != self.JpegEndSequence: + return False + return True + + + # Allows us to using the with: scope. + def __enter__(self): + return self + + + # Allows us to using the with: scope. + # Must not throw! + def __exit__(self, t, v, tb): + # Close in the opposite order they were opened. + try: + if self.PipeSelect is not None: + self.PipeSelect.close() + except Exception: + pass + try: + if self.Process is not None: + self.Process.kill() + except Exception: + pass From 530dba717c2b05dd18803efcc72e12e0ed7efe2a Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Fri, 22 Mar 2024 09:51:45 -0700 Subject: [PATCH 059/328] Adding logic to install or update ffmpeg for Bambu installs --- py_installer/Context.py | 19 ++++++++++--------- py_installer/Ffmpeg.py | 24 ++++++++++++++++++++++++ py_installer/Installer.py | 5 +++++ py_installer/Updater.py | 12 +++++++++++- setup.py | 2 +- 5 files changed, 51 insertions(+), 11 deletions(-) create mode 100644 py_installer/Ffmpeg.py diff --git a/py_installer/Context.py b/py_installer/Context.py index 0e8fb47..ce7a5d8 100644 --- a/py_installer/Context.py +++ b/py_installer/Context.py @@ -228,32 +228,33 @@ def ParseCmdLineArgs(self): # Handle and flags passed. if a[0] == '-': rawArg = a[1:] - if rawArg.lower() == "debug": + rawArgLower = rawArg.lower() + if rawArgLower == "debug": # Enable debug printing. self.Debug = True Logger.EnableDebugLogging() - elif rawArg.lower() == "help" or rawArg.lower() == "usage" or rawArg.lower() == "h": + elif rawArgLower == "help" or rawArgLower == "usage" or rawArgLower == "h": self.ShowHelp = True - elif rawArg.lower() == "skipsudoactions": + elif rawArgLower == "skipsudoactions": Logger.Warn("Skipping sudo actions. ! This will not result in a valid install! ") self.SkipSudoActions = True - elif rawArg.lower() == "noatuoselect": + elif rawArgLower == "noatuoselect": Logger.Info("Disabling Moonraker instance auto selection.") self.DisableAutoMoonrakerInstanceSelection = True - elif rawArg.lower() == "observer": + elif rawArgLower == "observer": # This is the legacy flag Logger.Info("Setup running in companion setup mode.") self.IsCompanionSetup = True - elif rawArg.lower() == "companion": + elif rawArgLower == "companion": Logger.Info("Setup running in companion setup mode.") self.IsCompanionSetup = True - elif rawArg.lower() == "bambu": + elif rawArgLower == "bambu": Logger.Info("Setup running in Bambu Connect setup mode.") self.IsBambuSetup = True - elif rawArg.lower() == "update": + elif rawArgLower == "update" or rawArgLower == "upgrade": Logger.Info("Setup running in update mode.") self.IsUpdateMode = True - elif rawArg.lower() == "uninstall": + elif rawArgLower == "uninstall": Logger.Info("Setup running in uninstall mode.") self.IsUninstallMode = True else: diff --git a/py_installer/Ffmpeg.py b/py_installer/Ffmpeg.py new file mode 100644 index 0000000..6136eeb --- /dev/null +++ b/py_installer/Ffmpeg.py @@ -0,0 +1,24 @@ +import time + +from .Util import Util +from .Logging import Logger + +# A helper class to make sure ffmpeg is installed. +class Ffmpeg: + + # Tries to install ffmpeg, but this won't fail if the install fails. + @staticmethod + def EnsureFfmpeg(): + # Try to install or upgrade. + Logger.Info("Installing ffmpeg, this might take a moment...") + startSec = time.time() + (returnCode, stdOut, stdError) = Util.RunShellCommand("sudo apt-get install ffmpeg -y", False) + + # Report the status to the installer log. + Logger.Debug(f"FFmpeg install result. Code: {returnCode}, StdOut: {stdOut}, StdErr: {stdError}") + if returnCode == 0: + Logger.Info(f"Ffmpeg successfully installed/updated. It took {str(round(time.time()-startSec, 2))} seconds.") + return + + # Warn, but don't throw or stop the installer. + Logger.Warn(f"Ffmpeg failed to install. It took {str(round(time.time()-startSec, 2))} seconds. Error: {stdError}") diff --git a/py_installer/Installer.py b/py_installer/Installer.py index 30cc306..6fa6025 100644 --- a/py_installer/Installer.py +++ b/py_installer/Installer.py @@ -13,6 +13,7 @@ from .TimeSync import TimeSync from .Frontend import Frontend from .Uninstall import Uninstall +from .Ffmpeg import Ffmpeg class Installer: @@ -139,6 +140,10 @@ def _RunInternal(self): frontend = Frontend() frontend.DoFrontendSetup(context) + # If this is a bambu setup, make sure ffmpeg is installed since it's required for the X1 webcam. + if context.IsBambuSetup: + Ffmpeg.EnsureFfmpeg() + # Before we start the service, check if the secrets config file already exists and if a printer id already exists. # This will indicate if this is a fresh install or not. context.ExistingPrinterId = Linker.GetPrinterIdFromServiceSecretsConfigFile(context) diff --git a/py_installer/Updater.py b/py_installer/Updater.py index 95613b6..614e3a2 100644 --- a/py_installer/Updater.py +++ b/py_installer/Updater.py @@ -10,6 +10,7 @@ from .Paths import Paths from .Service import Service from .Util import Util +from .Ffmpeg import Ffmpeg # # This class is responsible for doing updates for all local, companions, and bambu connect plugins on this local system. @@ -33,16 +34,25 @@ def DoUpdate(self, context:Context): # Note GetServiceFileFolderPath will return dynamically based on the OsType detected. # Use sorted, so the results are in a nice user presentable order. foundOeServices = [] + hasAnyBambuConnects = False fileAndDirList = sorted(os.listdir(Paths.GetServiceFileFolderPath(context))) for fileOrDirName in fileAndDirList: Logger.Debug(f"Searching for OE services to update, found: {fileOrDirName}") - if Configure.c_ServiceCommonName in fileOrDirName.lower(): + fileOrDirNameLower = fileOrDirName.lower() + if Configure.c_ServiceCommonName in fileOrDirNameLower: foundOeServices.append(fileOrDirName) + if "bambu" in fileOrDirNameLower: + hasAnyBambuConnects = True if len(foundOeServices) == 0: Logger.Warn("No local, companion, or Bambu Connect plugins were found on this device.") raise Exception("No local, companion, or Bambu Connect plugins were found on this device.") + # If this is a bambu system, we also want to make sure we install/upgrade ffmpeg + # Since it's required for the X1 camera streaming. + if hasAnyBambuConnects: + Ffmpeg.EnsureFfmpeg() + Logger.Info("We found the following plugins to update:") for s in foundOeServices: Logger.Info(f" {s}") diff --git a/setup.py b/setup.py index 45d7b72..d3fe8c5 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this is also parsed by the moonraker module to pull the version, so the string and format must remain the same! -plugin_version = "3.1.1" +plugin_version = "3.1.5" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 857edb3ce13b34b5ba9ef25102be59dfa213f073 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Fri, 22 Mar 2024 09:53:15 -0700 Subject: [PATCH 060/328] Version bump! --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d3fe8c5..81ba081 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this is also parsed by the moonraker module to pull the version, so the string and format must remain the same! -plugin_version = "3.1.5" +plugin_version = "3.2.0" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 7ca5c0162ef541ed9d987acfae3234024f7d6669 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Fri, 22 Mar 2024 14:45:56 -0700 Subject: [PATCH 061/328] More X1 Webcam Improvements --- bambu_octoeverywhere/quickcam.py | 142 +++++++++++++++++++++++++------ setup.py | 2 +- 2 files changed, 116 insertions(+), 28 deletions(-) diff --git a/bambu_octoeverywhere/quickcam.py b/bambu_octoeverywhere/quickcam.py index 5f1527d..cdc9f0d 100644 --- a/bambu_octoeverywhere/quickcam.py +++ b/bambu_octoeverywhere/quickcam.py @@ -143,24 +143,25 @@ def _captureThread(self): # Sleep for a bit. time.sleep(2.0) - # Create the camera implementation we need for this device. - camImpl = None - # Since we have to use the URL.... - # IF the URL is empty, it's an X1 with LAN streaming disabled. - # If the URL has an address, it's an X1 with LAN streaming. - # If it's None, it's a P1, A1, or another printer with no RTSP. - if rtspUrl is not None: - camImpl = QuickCam_RTSP(self.Logger) - else: - # Default to the websocket impl, since it's used on the most printers. - camImpl = QuickCam_WebSocket(self.Logger) - - # Wrap the usage into a with, so the connection is always cleaned up - with camImpl: - # We allow a few attempts, so if there are any connection issues or errors we buffer them out. - attempts = 0 - while attempts < 5: - attempts += 1 + # We allow a few attempts, so if there are any connection issues or errors we buffer them out. + attempts = 0 + while attempts < 5: + attempts += 1 + + # Create the camera implementation we need for this device. + camImpl = None + # Since we have to use the URL.... + # IF the URL is empty, it's an X1 with LAN streaming disabled. + # If the URL has an address, it's an X1 with LAN streaming. + # If it's None, it's a P1, A1, or another printer with no RTSP. + if rtspUrl is not None: + camImpl = QuickCam_RTSP(self.Logger) + else: + # Default to the websocket impl, since it's used on the most printers. + camImpl = QuickCam_WebSocket(self.Logger) + + # Wrap the usage into a with, so the connection is always cleaned up + with camImpl: try: # Connect to the server. camImpl.Connect(ipOrHostname, accessCode) @@ -320,6 +321,13 @@ def __exit__(self, t, v, tb): # Implements the websocket camera version for the X1 series printers. class QuickCam_RTSP: + # How long we will wait for data on each read before timing out. + c_ReadTimeoutSec = 5.0 + + # Adds a ton of logging useful for debugging. + c_DebugLogging = False + + def __init__(self, logger:logging.Logger): self.Logger = logger self.Process:subprocess.Popen = None @@ -331,15 +339,25 @@ def __init__(self, logger:logging.Logger): self.JpegStartSequenceLen = len(self.JpegStartSequence) self.JpegEndSequence = bytearray([0xff, 0xd9]) self.PipeSelect = selectors.DefaultSelector() + self.TimeSinceLastImg = time.time() + + # Std Error logic + self.StdErrBuffer = "" + self.ErrorReaderThread:threading.Thread = None + self.ErrorReaderThreadRunning = True # ~~ Interface Function ~~ # Connects to the server. # This will throw an exception if it fails. def Connect(self, ipOrHostname:str, accessCode:str) -> None: - # TODO check for ffmpeg - # TODO detect if ffmpeg has died or failed to run # TODO get the address from the bambu state object + + # We set the logging level of ffmpeg depending on our logging level + # The logs are written to stderr even if they aren't errors, which is nice, so + # we can capture them on timeouts. + logLevel = "trace" if self.Logger.isEnabledFor(logging.DEBUG) else "warning" + # Notes # We use 15 fps because it's a good trade off of fps and cpu perf hits # It also decreases the bandwidth needed, which helps on mobile @@ -348,7 +366,7 @@ def Connect(self, ipOrHostname:str, accessCode:str) -> None: self.Process = subprocess.Popen(["ffmpeg", "-hide_banner", "-y", - "-loglevel", "error", + "-loglevel", logLevel, "-rtsp_transport", "tcp", "-use_wallclock_as_timestamps", "1", "-i", f"rtsps://bblp:{accessCode}@{ipOrHostname}:322/streaming/live/1", @@ -359,8 +377,16 @@ def Connect(self, ipOrHostname:str, accessCode:str) -> None: stdout=subprocess.PIPE, stderr=subprocess.PIPE) # pylint: disable=no-member # Linux only os.set_blocking(self.Process.stdout.fileno(), False) + os.set_blocking(self.Process.stderr.fileno(), False) self.PipeSelect.register(self.Process.stdout, selectors.EVENT_READ) + # Since we setup the stderr pipe, we must read from it. If it fills it's buffer it will block the ffmpeg process. + self.ErrorReaderThread = threading.Thread(target=self._ErrorReader) + self.ErrorReaderThread.start() + + if QuickCam_RTSP.c_DebugLogging: + self.Logger.debug("Ffmpeg process started.") + # ~~ Interface Function ~~ # Gets an image from the server. This should block until an image is ready. @@ -369,21 +395,32 @@ def Connect(self, ipOrHostname:str, accessCode:str) -> None: def GetImage(self) -> bytearray: while True: # Wait on the pipe, which will signal us when there's data to be read. - self.PipeSelect.select() + # We timeout after 5 seconds, which is plenty of time for the stream to be ready. + self.PipeSelect.select(QuickCam_RTSP.c_ReadTimeoutSec) # Read all of the data we can. buffer = self.Process.stdout.read(100000000) + # Check for a timeout. This can happen because the select timeout, or it's been too long since we got an image parsed. + # This usually means that ffmpeg has died or is not running correctly. + if self.Process.returncode is not None or (time.time() - self.TimeSinceLastImg) > QuickCam_RTSP.c_ReadTimeoutSec: + if self.StdErrBuffer is None or len(self.StdErrBuffer) == 0: + self.StdErrBuffer = "" + raise Exception(f"Ffmpeg read timeout. StdError: {self.StdErrBuffer}") + # If we get an empty buffer, we just need to wait for more. if buffer is None or len(buffer) == 0: + if QuickCam_RTSP.c_DebugLogging: + self.Logger.debug("RTSP read empty buffer from stdin.") continue # If there's no pending buffered data do a quick exit if we were able to read the entire buffer in our first read. # If the full buffer is only one jpeg images, we don't need to do any scanning and can just return it. if self.Buffer is None: if self._CheckIfFullJpeg(buffer): - #self.Logger.info(f"quick image {len(buffer)}") - self._ResetLocalBuffer() + self._ResetLocalBuffer(True) + if QuickCam_RTSP.c_DebugLogging: + self.Logger.debug("RTSP fast path image received.") return buffer # Append this buffer to the current pending buffer. @@ -400,8 +437,9 @@ def GetImage(self) -> bytearray: # Check if the buffer is a full image now if self._CheckIfFullJpeg(self.Buffer): img = self.Buffer - self._ResetLocalBuffer() - #self.Logger.info("second quick exit") + self._ResetLocalBuffer(True) + if QuickCam_RTSP.c_DebugLogging: + self.Logger.debug("RTSP second quick path image received.") return img # Scan the buffer for the jpeg end sequence. @@ -421,15 +459,22 @@ def GetImage(self) -> bytearray: # So reset the buffer and continue. Note after we reset the buffer, we might # hit this, since we could have a partial image in the Buffer self._ResetLocalBuffer() + if QuickCam_RTSP.c_DebugLogging: + self.Logger.debug("RTSP we found a jpeg end sequence, but the buffer didn't start with a jpeg start sequence.") continue # Take the image off the buffer. self.Buffer = self.Buffer[newImageStart:] self.SearchedIndex = 0 + self.TimeSinceLastImg = time.time() # Ensure the buffer isn't too long. self._ResetLocalBufferIfOverLimit() + if QuickCam_RTSP.c_DebugLogging: + self.Logger.debug("RTSP slow path image received.") return imgBuffer # If we didn't find anything, check the limit. + if QuickCam_RTSP.c_DebugLogging: + self.Logger.debug("We got a new buffer with no image match.") self._ResetLocalBufferIfOverLimit() @@ -441,9 +486,41 @@ def _ResetLocalBufferIfOverLimit(self): self._ResetLocalBuffer() - def _ResetLocalBuffer(self): + def _ResetLocalBuffer(self, hasNewImage:bool = False): self.SearchedIndex = 0 self.Buffer = None + if hasNewImage: + self.TimeSinceLastImg = time.time() + + + # Reads the error stream from ffmpeg. + # Since we pipe the images via stdout, stderr will have all of the logs, not just errors. + def _ErrorReader(self): + while self.ErrorReaderThreadRunning: + try: + # Use a selector, so we only wake up when there's data to be read. + with selectors.DefaultSelector() as selector: + selector.register(self.Process.stderr, selectors.EVENT_READ) + while self.ErrorReaderThreadRunning: + # Wait for data to be read. + # We can wait on this forever, because when the processes closes, the pipe will close, and that will release the select call. + selector.select() + + # Check that we aren't shutting down. + if self.ErrorReaderThreadRunning is False: + return + + # Read the data. + buffer = self.Process.stderr.read(10000) + if buffer is not None and len(buffer) > 0: + # Append to the log + self.StdErrBuffer += buffer.decode("utf-8") + # Have some sanity limit + if len(self.StdErrBuffer) > 100000: + self.StdErrBuffer = self.StdErrBuffer[-100000:] + + except Exception as e: + Sentry.Exception("RTSP error reader thread failed.", e) # Checks if the buffer is only one image, from start to end. @@ -465,14 +542,25 @@ def __enter__(self): # Allows us to using the with: scope. # Must not throw! def __exit__(self, t, v, tb): + # Close the error reader thread. + # Killing the process will cause the error reader thread to exit. + self.ErrorReaderThreadRunning = False + # Close in the opposite order they were opened. try: if self.PipeSelect is not None: self.PipeSelect.close() except Exception: pass + # Tell the process to be killed try: if self.Process is not None: self.Process.kill() except Exception: pass + # And then call exit to cleanup all of the pipes and process handles. + try: + if self.Process is not None: + self.Process.__exit__(t, v, tb) + except Exception: + pass diff --git a/setup.py b/setup.py index 81ba081..053bb29 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this is also parsed by the moonraker module to pull the version, so the string and format must remain the same! -plugin_version = "3.2.0" +plugin_version = "3.2.1" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 845c959a8aa150c9fd247eeeb1b500110bcba779 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Fri, 22 Mar 2024 19:55:09 -0700 Subject: [PATCH 062/328] Adding logic for Bambu to detect IP address changes and follow the printer to the new IP --- bambu_octoeverywhere/bambuclient.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/bambu_octoeverywhere/bambuclient.py b/bambu_octoeverywhere/bambuclient.py index 23fe3b1..84fae70 100644 --- a/bambu_octoeverywhere/bambuclient.py +++ b/bambu_octoeverywhere/bambuclient.py @@ -134,12 +134,12 @@ def _ClientWorker(self): elif isinstance(e, TimeoutError): # This means there was no open socket at the given IP and port. self.Logger.error(f"Failed to connect to the Bambu printer {ipOrHostname}:{self.PortStr}, we will retry in a bit. "+str(e)) - elif isinstance(e, OSError) and "Network is unreachable" in str(e): + elif isinstance(e, OSError) and "Network is unreachable" in str(e) or "No route to host" in str(e): # This means the IP doesn't route to a device. self.Logger.error(f"Failed to connect to the Bambu printer {ipOrHostname}:{self.PortStr}, we will retry in a bit. "+str(e)) else: # Random other errors. - Sentry.Exception("FaWiled to connect to Bambu printer.", e) + Sentry.Exception("Failed to connect to the Bambu printer {ipOrHostname}:{self.PortStr}. We will retry in a bit.", e) # Sleep for a bit between tries. # The main consideration here is to not log too much when the printer is off. But we do still want to connect quickly, when it's back on. @@ -330,13 +330,17 @@ def _GetIpOrHostnameToTry(self) -> str: return configIpOrHostname # If we fail too many times, try to scan for the printer on the local subnet, the IP could have changed. - # Since we 100% identify the printer by the access token and printer SN, we can try to scan for it, and if we find something, it must be it. + # Since we 100% identify the printer by the access token and printer SN, we can try to scan for it. ips = NetworkSearch.ScanForInstances_Bambu(self.Logger, self.AccessToken, self.PrinterSn) - # If we find a different IP, try it. + # If we get an IP back, it is the printer. + # The scan above will only return an IP if the printer was successfully connected to, logged into, and fully authorized with the Access Token and Printer SN. if len(ips) == 1: + # Since we know this is the IP, we will update it in the config. This mean in the future we will use this IP directly + # And everything else trying to connect to the printer (webcam and ftp) will use the correct IP. ip = ips[0] - self.Logger.info(f"We found a possible IP for this instance {ip}, trying it now.") + self.Logger.info(f"We found a new IP for this printer. [{configIpOrHostname} -> {ip}] Updating the config and using it to connect.") + self.Config.SetStr(Config.SectionCompanion, Config.CompanionKeyIpOrHostname, ip) return ip # If we don't find anything, just use the config IP. From 9be34c6c274140ab00c45c6c021bffa1b45b8e81 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Fri, 22 Mar 2024 19:59:34 -0700 Subject: [PATCH 063/328] Version bump. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 053bb29..e177c7e 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this is also parsed by the moonraker module to pull the version, so the string and format must remain the same! -plugin_version = "3.2.1" +plugin_version = "3.2.2" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 5bcb79967ac78c81fc6245733676a8cededfc966 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Fri, 22 Mar 2024 21:27:20 -0700 Subject: [PATCH 064/328] Adding a guide to help enable the camera streaming on the x1. --- .vscode/settings.json | 2 + bambu_octoeverywhere/bambuclient.py | 3 +- linux_host/networksearch.py | 66 +++++++++++++++++-- .../NetworkConnectors/BambuConnector.py | 41 +++++++++++- setup.py | 2 +- 5 files changed, 105 insertions(+), 9 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 676450d..a252019 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -74,6 +74,7 @@ "INET", "inited", "ints", + "ipcam", "JFIF", "jmpeg", "journalctl", @@ -87,6 +88,7 @@ "Klippy", "levelname", "levelno", + "Liveview", "localauth", "localfs", "localip", diff --git a/bambu_octoeverywhere/bambuclient.py b/bambu_octoeverywhere/bambuclient.py index 84fae70..bfa7167 100644 --- a/bambu_octoeverywhere/bambuclient.py +++ b/bambu_octoeverywhere/bambuclient.py @@ -159,7 +159,7 @@ def _FullSyncWorker(): getInfo = {"info": {"sequence_id": "0", "command": "get_version"}} if not self._Publish(getInfo): raise Exception("Failed to publish get_version") - pushAll = { "pushing": {"sequence_id": "55", "command": "pushall"}} + pushAll = { "pushing": {"sequence_id": "0", "command": "pushall"}} if not self._Publish(pushAll): raise Exception("Failed to publish full sync") except Exception as e: @@ -262,6 +262,7 @@ def _OnMessage(self, client, userdata, mqttMsg:mqtt.MQTTMessage): if cmd is not None and cmd == "push_status": # We dont have a 100% great way to know if this is a fully sync message. # For now, we use this stat. The message we get from a P1P has 59 members in the root, so we use 40 as mark. + # Note we use this same value in NetworkSearch.ValidateConnection_Bambu if len(printMsg) > 40: isFirstFullSyncResponse = True self.HasDoneFirstFullStateSync = True diff --git a/linux_host/networksearch.py b/linux_host/networksearch.py index 76a4407..63d7648 100644 --- a/linux_host/networksearch.py +++ b/linux_host/networksearch.py @@ -1,4 +1,5 @@ import ssl +import json import socket import logging import threading @@ -7,11 +8,16 @@ import paho.mqtt.client as mqtt class NetworkValidationResult: - def __init__(self, failedToConnect:bool = False, failedAuth:bool = False, failSn:bool = False, exception:Exception = None) -> None: + def __init__(self, failedToConnect:bool = False, failedAuth:bool = False, failSn:bool = False, exception:Exception = None, bambuRtspUrl = None) -> None: self.FailedToConnect = failedToConnect self.FailedAuth = failedAuth self.FailedSerialNumber = failSn self.Exception = exception + # Only used for Bambu printers. + # If none, the printer doesn't support RTSP. + # If empty string, the LAN Mode Liveview is not turned on. + # If a URL, the printer is ready to stream. + self.BambuRtspUrl = bambuRtspUrl def Success(self) -> bool: @@ -33,6 +39,16 @@ def callback(ip:str): return NetworkSearch._ScanForInstances(logger, callback) + # The final two steps can happen in different orders, so we need to wait for both the sub success and state object to be received. + @staticmethod + def _BambuConnectionDone(data:dict, client:mqtt.Client) -> bool: + if "SnSubSuccess" in data and data["SnSubSuccess"] is True and "GotStateObj" in data and data["GotStateObj"] is True: + data["Event"].set() + client.disconnect() + return True + return False + + # Given the ip, accessCode, printerSn, and optionally port, this will check if the printer is connectable. # Returns a NetworkValidationResult with the results. @staticmethod @@ -85,15 +101,47 @@ def subscribe(client, userdata:dict, mid, reason_code_list:List[mqtt.ReasonCode] logger.debug(f"Bambu {ipOrHostname} Sub response for the report subscription reports failure. {r}") failedSn = True if not failedSn: + # Note we are now subed. userdata["SnSubSuccess"] = True - logger.debug(f"Bambu {ipOrHostname} Sub success, the serial number is good") - client.disconnect() - userdata["Event"].set() + logger.debug(f"Bambu {ipOrHostname} Sub success.") + # Push the message to get the full state, this is needed on teh P1 and A1 + client.publish(f"device/{printerSn}/report", json.dumps( { "pushing": {"sequence_id": "0", "command": "pushall"}})) + # Check if we are done, this will disconnect if we are. + NetworkSearch._BambuConnectionDone(userdata, client) + + def message(client, userdata:dict, mqttMsg:mqtt.MQTTMessage): + # When we get a message, check if it is a state object. + # We need info from the state object, and also it's a good validation the system is healthy. + try: + msg = json.loads(mqttMsg.payload) + if "print" in msg: + printMsg = msg["print"] + # We dont have a 100% great way to know if this is a fully sync message. + # For now, we use this stat. The message we get from a P1P has 59 members in the root, so we use 40 as mark. + # Note we use this same value in BambuClient._OnMessage + if len(printMsg) > 40: + # Indicate we got the state object. + userdata["GotStateObj"] = True + # Try to parse the rtsp url if the printer has one. + ipCam = printMsg.get("ipcam", None) + rtspUrl = None + if ipCam is not None: + rtspUrl = ipCam.get("rtsp_url", None) + userdata["BambuRtspUrl"] = rtspUrl + # Report we got the full sync object and see if we are done. + logger.debug(f"Bambu {ipOrHostname} got a full state sync message. RTSP URL: {rtspUrl}") + # Check if we are done, this will disconnect if we are. + NetworkSearch._BambuConnectionDone(userdata, client) + else: + logger.debug(f"Bambu {ipOrHostname} got a state message, but it was too small to be a full message.") + except Exception as e: + logger.debug(f"Bambu {ipOrHostname} - message failure {e}") # Setup functions and connect. client.on_connect = connect client.on_disconnect = disconnect client.on_subscribe = subscribe + client.on_message = message # Try to connect, this will throw if it fails to find any server to connect to. failedToConnect = True @@ -111,12 +159,18 @@ def subscribe(client, userdata:dict, mid, reason_code_list:List[mqtt.ReasonCode] # Walk though the connection and see how far we got. failedAuth = True failedSn = True + rtspUrl = None + if "IsAuthorized" in result: failedAuth = False - if "SnSubSuccess" in result: + # We need both the sub success message and a successful state sync to consider this success. + if "SnSubSuccess" in result and "GotStateObj" in result: failedSn = False + # Optional - Get the URL if there was one detected. + if "BambuRtspUrl" in result: + rtspUrl = result["BambuRtspUrl"] - return NetworkValidationResult(failedToConnect, failedAuth, failedSn) + return NetworkValidationResult(failedToConnect, failedAuth, failedSn, bambuRtspUrl=rtspUrl) except Exception as e: return NetworkValidationResult(exception=e) diff --git a/py_installer/NetworkConnectors/BambuConnector.py b/py_installer/NetworkConnectors/BambuConnector.py index 1be2b5e..8e1a6a5 100644 --- a/py_installer/NetworkConnectors/BambuConnector.py +++ b/py_installer/NetworkConnectors/BambuConnector.py @@ -1,4 +1,4 @@ -from linux_host.networksearch import NetworkSearch +from linux_host.networksearch import NetworkSearch, NetworkValidationResult from py_installer.Util import Util from py_installer.Logging import Logger @@ -23,6 +23,8 @@ def EnsureBambuConnection(self, context:Context): result = NetworkSearch.ValidateConnection_Bambu(Logger.GetPyLogger(), ip, accessCode, printerSn, portStr=port, timeoutSec=10.0) if result.Success(): Logger.Info("Successfully connected to you Bambu Lab printer!") + # Ensure the X1 camera is setup. + self._EnsureX1CameraSetup(result) return else: # Let the user keep this connection setup, or try to set it up again. @@ -35,6 +37,9 @@ def EnsureBambuConnection(self, context:Context): ipOrHostname, port, accessToken, printerSn = self._SetupNewBambuConnection() Logger.Info(f"You Bambu printer was found and authentication was successful! IP: {ipOrHostname}") + # Ensure the X1 camera is setup. + self._EnsureX1CameraSetup(None, ipOrHostname, accessToken, printerSn) + ConfigHelper.WriteCompanionDetails(context, ipOrHostname, port) ConfigHelper.WriteBambuDetails(context, accessToken, printerSn) Logger.Blank() @@ -214,3 +219,37 @@ def _SetupNewBambuConnection(self): # Default to a full restart. # Breaking this loop will return us to the main setup loop, which will do the entire access code and sn entry again. break + + + # Given either a validation result or required details, this gets the x1 carbon's ip camera state + # and ensures it's enabled properly + def _EnsureX1CameraSetup(self, result:NetworkValidationResult = None, ipOrHostname:str = None, accessToken:str = None, printerSn:str = None): + # If we didn't get passed a result, get one now. + if result is None: + result = NetworkSearch.ValidateConnection_Bambu(Logger.GetPyLogger(), ipOrHostname, accessToken, printerSn) + # If we didn't get something, it's a failure. + if result is None: + Logger.Error("Ensure camera failed to get a validation result.") + return + # Ensure success. + if result.Success() is False: + Logger.Error("Ensure camera got a validation result that failed.") + return + # If the bambu rtsp url is None, this printer doesn't support RTSP. + if result.BambuRtspUrl is None: + Logger.Info("This printer doesn't use RTSP camera streaming, skipping setup.") + return + # If there is a string and the string isn't 'disable' then we are good to go. + if len(result.BambuRtspUrl) > 0 and "disable" not in result.BambuRtspUrl: + Logger.Info(f"RTSP Liveview is enabled. '{result.BambuRtspUrl}'") + return + # Tell the user how to enable it. + Logger.Info("") + Logger.Info("") + Logger.Header("You need to enable 'LAN Mode Liveview' on your printer for webcam access.") + Logger.Blank() + Logger.Warn("Don't worry, you just need to flip a switch in the settings on your printer.") + Logger.Warn("Follow this link for a step-by-step guide:") + Logger.Warn("https://octoeverywhere.com/s/liveview") + Logger.Blank() + Util.AskYesOrNoQuestion("Did you enable 'LAN Mode Liveview'?") diff --git a/setup.py b/setup.py index e177c7e..7c98b16 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this is also parsed by the moonraker module to pull the version, so the string and format must remain the same! -plugin_version = "3.2.2" +plugin_version = "3.2.3" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From d033ecf2bd054c940cbc5b5155e717fab4186558 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Fri, 22 Mar 2024 21:36:40 -0700 Subject: [PATCH 065/328] Lint fix. --- .pylintrc | 1 + 1 file changed, 1 insertion(+) diff --git a/.pylintrc b/.pylintrc index 09fc441..e108966 100644 --- a/.pylintrc +++ b/.pylintrc @@ -25,6 +25,7 @@ disable= R1730, # consider-using-min-builtin R1731, # consider-using-max-builtin R1724, # Unnecessary "else" after "break" + R1715, # consider-using-get for dict gets. # A comma-separated list of package or module names from where C extensions may From 516bde8cd435632714101b1b5e2976051363bd76 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sun, 24 Mar 2024 15:07:59 -0700 Subject: [PATCH 066/328] Perf optimizations for the plugin! --- moonraker_octoeverywhere/moonrakerclient.py | 4 +- .../WebStream/octowebstreamhttphelper.py | 36 ++-- .../WebStream/octowebstreamwshelper.py | 30 ++- octoeverywhere/octoservercon.py | 6 +- octoeverywhere/websocketimpl.py | 181 +++++++++++++----- .../NetworkConnectors/MoonrakerConnector.py | 35 ++-- 6 files changed, 205 insertions(+), 87 deletions(-) diff --git a/moonraker_octoeverywhere/moonrakerclient.py b/moonraker_octoeverywhere/moonrakerclient.py index 3b8ca63..263638f 100644 --- a/moonraker_octoeverywhere/moonrakerclient.py +++ b/moonraker_octoeverywhere/moonrakerclient.py @@ -508,7 +508,9 @@ def _WebSocketWorkerThread(self): ) # Run until the socket closes - self.WebSocket.RunUntilClosed() + # When it returns, ensure it's closed. + with self.WebSocket: + self.WebSocket.RunUntilClosed() except Exception as e: Sentry.Exception("Moonraker client exception in main WS loop.", e) diff --git a/octoeverywhere/WebStream/octowebstreamhttphelper.py b/octoeverywhere/WebStream/octowebstreamhttphelper.py index b2549ac..f874b33 100644 --- a/octoeverywhere/WebStream/octowebstreamhttphelper.py +++ b/octoeverywhere/WebStream/octowebstreamhttphelper.py @@ -524,10 +524,11 @@ def copyUploadDataFromMsg(self, webStreamMsg): # Done! return + # NOTE: We can't do this! Since we try to compress all of the things right now, for already compressed things it will add a little overhead! + # The full upload size will be the same size as we expect, but the compression will make the payload larger. # If we know the upload size, make sure this doesn't exceeded it. - if self.KnownFullStreamUploadSizeBytes is not None and thisMessageDataLen + self.UploadBytesReceivedSoFar > self.KnownFullStreamUploadSizeBytes: - self.Logger.warn(self.getLogMsgPrefix() + " received more bytes than it was expecting for the upload. thisMsg:"+str(thisMessageDataLen)+"; so far:"+str(self.UploadBytesReceivedSoFar) + "; expected:"+str(self.KnownFullStreamUploadSizeBytes)) - raise Exception("Too many bytes received for http upload buffer") + # if self.KnownFullStreamUploadSizeBytes is not None and thisMessageDataLen + self.UploadBytesReceivedSoFar > self.KnownFullStreamUploadSizeBytes: + # self.Logger.warn(self.getLogMsgPrefix() + " received more bytes than it was expecting for the upload. thisMsg:"+str(thisMessageDataLen)+"; so far:"+str(self.UploadBytesReceivedSoFar) + "; expected:"+str(self.KnownFullStreamUploadSizeBytes)) # Make sure the array has been allocated and it's still large enough. if self.UploadBuffer is None or thisMessageDataLen + self.UploadBytesReceivedSoFar > len(self.UploadBuffer): @@ -554,6 +555,11 @@ def copyUploadDataFromMsg(self, webStreamMsg): # Get a slice of the buffer to avoid a the copy, since we copy on the next step anyways. buf = self.decompressBufferIfNeeded(webStreamMsg) + # Now that we have the original size of the body back, check to make sure it's not too much. + if self.KnownFullStreamUploadSizeBytes is not None and len(buf) + self.UploadBytesReceivedSoFar > self.KnownFullStreamUploadSizeBytes: + self.Logger.warn(self.getLogMsgPrefix() + " received more bytes than it was expecting for the upload. thisMsg:"+str(len(buf))+"; so far:"+str(self.UploadBytesReceivedSoFar) + "; expected:"+str(self.KnownFullStreamUploadSizeBytes)) + raise Exception("Too many bytes received for http upload buffer") + # Append the data into the main buffer. pos = self.UploadBytesReceivedSoFar self.UploadBuffer[pos:pos+len(buf)] = buf @@ -639,9 +645,11 @@ def shouldCompressBody(self, contentTypeLower, octoHttpResult, contentLengthOpt) if contentLengthOpt is not None and contentLengthOpt < 200: return False - # If we don't know what this is, might as well compress it. + # If we don't know what this is, we don't want to compress it. + # Compressing the body of a compressed thing will make it larger and takes a good amount of time, + # so we don't want to waste time on it. if contentTypeLower is None: - return True + return False # If there is a full body buffer and and it's already compressed, always return true. # This ensures the message is flagged correctly for compression and the body reading system @@ -655,21 +663,21 @@ def shouldCompressBody(self, contentTypeLower, octoHttpResult, contentLengthOpt) # - Anything that says it's json # - Anything that's xml # - Anything that's svg - # - Anything that's octet-stream. (we do this because some of the .less files don't get set as text correctly) return (contentTypeLower.find("text/") != -1 or contentTypeLower.find("javascript") != -1 or contentTypeLower.find("json") != -1 or contentTypeLower.find("xml") != -1 - or contentTypeLower.find("svg") != -1 or contentTypeLower.find("octet-stream") != -1) + or contentTypeLower.find("svg") != -1) # Reads data from the response body, puts it in a data vector, and returns the offset. # If the body has been fully read, this should return ogLen == 0, len = 0, and offset == None # The read style depends on the presence of the boundary string existing. def readContentFromBodyAndMakeDataVector(self, builder, octoHttpResult:OctoHttpRequest.Result, boundaryStr_opt, shouldCompress, contentTypeLower_NoneIfNotKnown, contentLength_NoneIfNotKnown, responseHandlerContext): - # This is the max size each body read will be. Since we are making local calls, most of the time - # we will always get this full amount as long as theres more body to read. - # Note that this amount is larger than a single read of the websocket on the server. After some testing - # we found the transfer was most efficient if we sent larger message sizes, because it could saturate the tcp link better. - defaultBodyReadSizeBytes = 199 * 1024 + # This is the max size each body read will be. Since we are making local calls, most of the time we will always get this full amount as long as theres more body to read. + # This size is a little under the max read buffer on the server, allowing the server to handle the buffers with no copies. + # + # 3/24/24 - We did a lot of direct download testing to tweak this buffer size and the server read size, these were the best values able to hit about 160mpbs. + # With the current values, the majority of the time is spent sending the data on the websocket. + defaultBodyReadSizeBytes = 490 * 1024 # If we are going to compress this read, use a much higher number. Since most of what we compress is text, # and that text usually compresses down to 25% of the og size, we will use a x4 multiplier. @@ -843,7 +851,9 @@ def readStreamChunk(self, octoHttpResult:OctoHttpRequest.Result, boundaryStr): # Read a small chunk to try to read the header # We want to read enough that hopefully we get all of the headers, but not so much that # we accidentally read two boundary messages at once. - headerBuffer = self.doBodyRead(octoHttpResult, 300) + # 3/24/24 - After a lot of testing, it seems most times we get the full headers in 120 chars. + # So we will target that much, hoping we can do one read and get them. + headerBuffer = self.doBodyRead(octoHttpResult, 120) # If this returns 0, the body read is complete if headerBuffer is None: diff --git a/octoeverywhere/WebStream/octowebstreamwshelper.py b/octoeverywhere/WebStream/octowebstreamwshelper.py index 3797a53..619a2fa 100644 --- a/octoeverywhere/WebStream/octowebstreamwshelper.py +++ b/octoeverywhere/WebStream/octowebstreamwshelper.py @@ -190,8 +190,23 @@ def AttemptConnection(self): # Make the websocket object and start it running. self.Logger.debug(self.getLogMsgPrefix()+"opening websocket to "+str(uri) + " attempt "+ str(self.ConnectionAttempt)) - self.Ws = Client(uri, self.onWsOpened, None, self.onWsData, self.onWsClosed, self.onWsError, subProtocolList=self.SubProtocolList) - self.Ws.RunAsync() + ws = Client(uri, self.onWsOpened, None, self.onWsData, self.onWsClosed, self.onWsError, subProtocolList=self.SubProtocolList) + + # To ensure we never leak a websocket, we need to use this lock. + # We need to check the is closed flag and then only set the ws if it's not closed. + with self.StateLock: + if self.IsClosed: + # Cleanup and leave + try: + ws.Close() + except Exception: + pass + return False + + # We aren't closed, set the websocket and run it. + # We have to be careful with this ws, because it needs to be closed to fully shutdown, but we can't use a with statement. + self.Ws = ws + self.Ws.RunAsync() # Return true to indicate we are trying to connect again. return True @@ -201,6 +216,7 @@ def AttemptConnection(self): # Called by the main socket thread so this should be quick! def Close(self): # Don't try to close twice. + wsToClose = None with self.StateLock: # If we are already closed, there's nothing to do. if self.IsClosed is True: @@ -208,16 +224,18 @@ def Close(self): # We will close now, so set the flag. self.IsClosed = True + # We use this lock to protect the websocket and make sure we never open when when we are closed or closing. + # We must capture this in the same lock as self.IsClosed + wsToClose = self.Ws + self.Logger.info(self.getLogMsgPrefix()+"websocket closed after " +str(time.time() - self.OpenedTime) + " seconds") # The initial connection is created (or at least started) in the constructor, but there's re-attempt logic # that can cause the websocket to be destroyed and re-created. For that reason we need to grab a local ref # and make sure it's not null. If the close fails, just ignore it, since we are shutting down already. - ws = self.Ws - self.Ws = None - if ws is not None: + if wsToClose is not None: try: - ws.Close() + wsToClose.Close() except Exception as _ : pass diff --git a/octoeverywhere/octoservercon.py b/octoeverywhere/octoservercon.py index b2f2b2b..02531c9 100644 --- a/octoeverywhere/octoservercon.py +++ b/octoeverywhere/octoservercon.py @@ -317,9 +317,11 @@ def RunBlocking(self): endpoint = self.GetEndpoint() # Connect to the service. + # When this returns, make sure it's fully closed. self.Ws = Client(endpoint, self.OnOpened, self.OnMsg, None, self.OnClosed, self.OnError) - self.Logger.info("Attempting to talk to OctoEverywhere, server con "+self.GetConnectionString() + " wsId:"+self.GetWsId(self.Ws)) - self.Ws.RunUntilClosed() + with self.Ws: + self.Logger.info("Attempting to talk to OctoEverywhere, server con "+self.GetConnectionString() + " wsId:"+self.GetWsId(self.Ws)) + self.Ws.RunUntilClosed() # Handle disconnects self.Logger.info("Disconnected from OctoEverywhere, server con "+self.GetConnectionString()) diff --git a/octoeverywhere/websocketimpl.py b/octoeverywhere/websocketimpl.py index 44b9f35..b9af175 100644 --- a/octoeverywhere/websocketimpl.py +++ b/octoeverywhere/websocketimpl.py @@ -1,3 +1,4 @@ +import queue import threading import certifi import websocket @@ -16,6 +17,11 @@ def __init__(self, url, onWsOpen = None, onWsMsg = None, onWsData = None, onWsCl self.wsErrorCallbackLock = threading.Lock() self.hasFiredWsErrorCallback = False + # We use a send queue thread because it allows us to process downloads about 2x faster. + # This is because the downstream work of the WS can be made faster if it's done in parallel + self.SendQueue = queue.Queue() + self.SendThread:threading.Thread = None + # Used to log more details about what's going on with the websocket. # websocket.enableTrace(True) @@ -23,6 +29,11 @@ def __init__(self, url, onWsOpen = None, onWsMsg = None, onWsData = None, onWsCl # any errors. self.hasClientRequestedClose = False + # This is used to keep track of this object has been closed. + # If this flag is true, this object should not be running and will never run again. + self.isClosed = False + self.isClosedLock = threading.Lock() + def OnOpen(ws): if onWsOpen: onWsOpen(self) @@ -58,51 +69,13 @@ def OnError(ws, exception): ) - # This can be called from our logic internally in this class or from - # The websocket class itself - def handleWsError(self, exception): - # If the client is trying to close this websocket and has made the close call to do so, - # we won't fire any more errors out of it. This can happen if a send is trying to send data - # at the same time as the socket is closing for example. - if self.hasClientRequestedClose: - return - - # Since this callback can be fired from many sources, we want to ensure it only - # gets fired once. - with self.wsErrorCallbackLock: - if self.hasFiredWsErrorCallback: - return - self.hasFiredWsErrorCallback = True - - # To prevent locking issues or other issues, spin off a thread to fire the callback. - # This prevents the case where send() fires the callback, we don't want to overlap the - # send path callback. - callbackThread = threading.Thread(target=self.fireWsErrorCallbackThread, args=(exception, )) - callbackThread.start() - - - def fireWsErrorCallbackThread(self, exception): - try: - # Fire the error callback. - if self.clientWsErrorCallback: - self.clientWsErrorCallback(self, exception) - - # Now ensure the websocket is closing. Since it most likely already is, - # ignore any exceptions. - try: - self.Ws.close() - except Exception as e: - # This is a known bug in the websocket class, it happens when the WS is closing. - if isinstance(e, AttributeError) and "object has no attribute 'close'" in str(e): - # We don't have a logger, sooooooo - print("Websocket closed due to: 'NoneType' object has no attribute 'close'") - else: - Sentry.Exception("Websocket fireWsErrorCallbackThread close exception", e) - except Exception as e : - Sentry.Exception("Websocket client exception in fireWsErrorCallbackThread", e) - - + # Runs the websocket blocking until it closes. def RunUntilClosed(self): + # Start the send queue thread if it hasn't been started. + if self.SendThread is None: + self.SendThread = threading.Thread(target=self._SendQueueThread, daemon=True) + self.SendThread.start() + # # The client is responsible for sending keep alive pings the server will then pong respond to. # If that's not done, the connection will timeout. We will send a ping every 10 minutes. @@ -118,6 +91,11 @@ def RunUntilClosed(self): # where it will call select() after the socket is closed, which makes select() hang until the time expires. # Thus we need to keep the ping_timeout low, so when this happens, it doesn't hang forever. try: + # Since some clients use RunAsync, check that we didn't close before the async action started. + with self.isClosedLock: + if self.isClosed: + return + self.Ws.run_forever(skip_utf8_validation=True, ping_interval=600, ping_timeout=20, sslopt={"ca_certs":certifi.where()}) except Exception as e: # There's a compat issue where run_forever will try to access "isAlive" when the socket is closing @@ -130,19 +108,79 @@ def RunUntilClosed(self): self.handleWsError(e) + # Runs the websocket async. def RunAsync(self): t = threading.Thread(target=self.RunUntilClosed, args=()) t.daemon = True t.start() + # Closes the websocket. def Close(self): self.hasClientRequestedClose = True # Always try to call close, even if we have already done it. + self._Close() + + + # Internally used to close and cleanup. + def _Close(self): + + # Set that we are now closed. + with self.isClosedLock: + self.isClosed = True + + # Always try to call close, even if we have already done it. + # Now ensure the websocket is closing. Since it most likely already is, ignore any exceptions. try: self.Ws.close() except Exception as e: - Sentry.Exception("Websocket close exception", e) + # This is a known bug in the websocket class, it happens when the WS is closing. + if isinstance(e, AttributeError) and "object has no attribute 'close'" in str(e): + # We don't have a logger, sooooooo + print("Websocket closed due to: 'NoneType' object has no attribute 'close'") + else: + Sentry.Exception("Websocket fireWsErrorCallbackThread close exception", e) + + # Always ensure we close the send queue. + try: + # Push an empty buffer to the send queue, which will close it. + self.SendQueue.put(SendQueueContext(None, None)) + except Exception as e: + Sentry.Exception("Exception while trying to close the send queue.", e) + + + # This can be called from our logic internally in this class or from the websocket class itself + def handleWsError(self, exception): + # If the client is trying to close this websocket and has made the close call to do so, + # we won't fire any more errors out of it. This can happen if a send is trying to send data + # at the same time as the socket is closing for example. + if self.hasClientRequestedClose: + return + + # Since this callback can be fired from many sources, we want to ensure it only + # gets fired once. + with self.wsErrorCallbackLock: + if self.hasFiredWsErrorCallback: + return + self.hasFiredWsErrorCallback = True + + # To prevent locking issues or other issues, spin off a thread to fire the callback. + # This prevents the case where send() fires the callback, we don't want to overlap the + # send path callback. + callbackThread = threading.Thread(target=self.fireWsErrorCallbackThread, args=(exception, )) + callbackThread.start() + + + def fireWsErrorCallbackThread(self, exception): + try: + # Fire the error callback. + if self.clientWsErrorCallback: + self.clientWsErrorCallback(self, exception) + except Exception as e : + Sentry.Exception("Websocket client exception in fireWsErrorCallbackThread", e) + + # Be sure we always close the WS + self._Close() def Send(self, msgBytes, isData): @@ -154,11 +192,56 @@ def Send(self, msgBytes, isData): def SendWithOptCode(self, msgBytes, opcode): try: - self.Ws.send(msgBytes, opcode) + # Make sure we have a buffer, this is invalid and it will also shutdown our send thread. + if msgBytes is None: + raise Exception("We tired to send a message to the websocket with a None buffer.") + self.SendQueue.put(SendQueueContext(msgBytes, opcode)) + except Exception as e: + # If any exception happens during sending, we want to report the error + # and shutdown the entire websocket. + self.handleWsError(e) + + + def _SendQueueThread(self): + try: + while self.isClosed is False: + # Wait on something to send. + context = self.SendQueue.get() + # If it's None, that means we are shutting down. + if context is None or context.Buffer is None: + return + # Send it! + self.Ws.send(context.Buffer, context.OptCode) except Exception as e: # If any exception happens during sending, we want to report the error # and shutdown the entire websocket. self.handleWsError(e) + finally: + # When the send queue closes, make sure the websocket is closed. + # This is a saftey, incase for some reason the websocket was open and we were told to close. + self._Close() + + + + # Support using with: + def __enter__(self): + return self + + + # Support using with; + def __exit__(self, exc_type, exc_value, traceback): + self.Close() + + + # When the object is deleted, make sure the threads are cleaned up. + def __del__(self): + try: + if self.Ws is not None and self.Ws.keep_running: + print("THIS SHOULD NEVER HAPPEN! Websocket was deleted without being closed.") + # Ensure we are fully closed. + self.Close() + except Exception: + pass # A helper for dealing with common websocket connection exceptions. @@ -192,3 +275,9 @@ def IsCommonConnectionException(e:Exception): except Exception: pass return False + + +class SendQueueContext(): + def __init__(self, buffer, optCode) -> None: + self.Buffer = buffer + self.OptCode = optCode diff --git a/py_installer/NetworkConnectors/MoonrakerConnector.py b/py_installer/NetworkConnectors/MoonrakerConnector.py index e2e4623..8f7a3d6 100644 --- a/py_installer/NetworkConnectors/MoonrakerConnector.py +++ b/py_installer/NetworkConnectors/MoonrakerConnector.py @@ -165,26 +165,23 @@ def OnError(ws, exception): # Create the websocket Logger.Debug(f"Checking for moonraker using the address: `{url}`") - ws = Client(url, onWsOpen=OnOpened, onWsMsg=OnMsg, onWsError=OnError, onWsClose=OnClosed) - ws.RunAsync() - - # Wait for the event or a timeout. - doneEvent.wait(timeoutSec) - - # Get the results before we close. - capturedSuccess = False - capturedEx = None - with lock: - if result.get("success", None) is not None: - capturedSuccess = result["success"] - if result.get("exception", None) is not None: - capturedEx = result["exception"] - - # Ensure the ws is closed try: - ws.Close() - except Exception: - pass + with Client(url, onWsOpen=OnOpened, onWsMsg=OnMsg, onWsError=OnError, onWsClose=OnClosed) as ws: + ws.RunAsync() + + # Wait for the event or a timeout. + doneEvent.wait(timeoutSec) + + # Get the results before we close. + capturedSuccess = False + capturedEx = None + with lock: + if result.get("success", None) is not None: + capturedSuccess = result["success"] + if result.get("exception", None) is not None: + capturedEx = result["exception"] + except Exception as e: + Logger.Info(f"Websocket threw and exception. {e}") return (capturedSuccess, capturedEx) From ea2f5197d44e3744e7d09c2ae69993101e6b83de Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sun, 24 Mar 2024 21:06:02 -0700 Subject: [PATCH 067/328] Disabling compression for streams that aren't going well with it. --- octoeverywhere/WebStream/octowebstreamhttphelper.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/octoeverywhere/WebStream/octowebstreamhttphelper.py b/octoeverywhere/WebStream/octowebstreamhttphelper.py index f874b33..2595f60 100644 --- a/octoeverywhere/WebStream/octowebstreamhttphelper.py +++ b/octoeverywhere/WebStream/octowebstreamhttphelper.py @@ -313,6 +313,15 @@ def executeHttpRequest(self): # Before we process the response, make sure we shouldn't defer for a high pri request self.checkForDelayIfNotHighPri() + # This is an interesting check. If we are spinning to deliver a http body, and we detect that what we are compressing + # is larger than the OG body, we will disable compression for all future messages. We do this because any files that's already + # compressed (video, audio, images, or files) will be the same after compression but with overhead added. + # We take a big time hit applying the compression, which is usually offset by the size reduction, but if that's not the case, disable it. + # If the compressed stream size (contentReadBytes) is larger than 90% of the original stream size(nonCompressedContentReadSizeBytes), stop compression. + if compressBody and contentReadBytes != 0 and nonCompressedContentReadSizeBytes != 0 and contentReadBytes > nonCompressedContentReadSizeBytes * 0.9: + compressBody = False + self.Logger.info(f"We detected that the compression being applied to this stream was inefficient, so we are disabling compression. Compression: {float(contentReadBytes)/float(nonCompressedContentReadSizeBytes)} URL: {uri}") + # Prepare a response. # TODO - We should start the buffer at something that's likely to not need expanding for most requests. builder = OctoStreamMsgBuilder.CreateBuffer(20000) @@ -675,7 +684,7 @@ def readContentFromBodyAndMakeDataVector(self, builder, octoHttpResult:OctoHttpR # This is the max size each body read will be. Since we are making local calls, most of the time we will always get this full amount as long as theres more body to read. # This size is a little under the max read buffer on the server, allowing the server to handle the buffers with no copies. # - # 3/24/24 - We did a lot of direct download testing to tweak this buffer size and the server read size, these were the best values able to hit about 160mpbs. + # 3/24/24 - We did a lot of direct download testing to tweak this buffer size and the server read size, these were the best values able to hit about 223mpbs. # With the current values, the majority of the time is spent sending the data on the websocket. defaultBodyReadSizeBytes = 490 * 1024 From 5776aef99d08831e27890d05d7da90f8b03fb1f6 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Thu, 28 Mar 2024 18:12:52 -0700 Subject: [PATCH 068/328] Fixing a few small bug and some issues with the X1 --- bambu_octoeverywhere/bambuclient.py | 8 ++++-- bambu_octoeverywhere/bambumodels.py | 29 ++++++++++++++++---- bambu_octoeverywhere/bambustatetranslater.py | 11 ++++++-- bambu_octoeverywhere/quickcam.py | 3 +- moonraker_octoeverywhere/uiinjector.py | 10 +++++-- octoeverywhere/notificationshandler.py | 2 ++ octoeverywhere/webcamhelper.py | 20 ++++---------- setup.py | 2 +- 8 files changed, 58 insertions(+), 27 deletions(-) diff --git a/bambu_octoeverywhere/bambuclient.py b/bambu_octoeverywhere/bambuclient.py index bfa7167..b327f60 100644 --- a/bambu_octoeverywhere/bambuclient.py +++ b/bambu_octoeverywhere/bambuclient.py @@ -1,8 +1,9 @@ import logging import ssl +import time import json +import socket import threading -import time from typing import List import paho.mqtt.client as mqtt @@ -134,9 +135,12 @@ def _ClientWorker(self): elif isinstance(e, TimeoutError): # This means there was no open socket at the given IP and port. self.Logger.error(f"Failed to connect to the Bambu printer {ipOrHostname}:{self.PortStr}, we will retry in a bit. "+str(e)) - elif isinstance(e, OSError) and "Network is unreachable" in str(e) or "No route to host" in str(e): + elif isinstance(e, OSError) and ("Network is unreachable" in str(e) or "No route to host" in str(e)): # This means the IP doesn't route to a device. self.Logger.error(f"Failed to connect to the Bambu printer {ipOrHostname}:{self.PortStr}, we will retry in a bit. "+str(e)) + elif isinstance(e, socket.timeout) and "timed out" in str(e): + # This means the IP doesn't route to a device. + self.Logger.error(f"Failed to connect to the Bambu printer {ipOrHostname}:{self.PortStr} due to a timeout, we will retry in a bit. "+str(e)) else: # Random other errors. Sentry.Exception("Failed to connect to the Bambu printer {ipOrHostname}:{self.PortStr}. We will retry in a bit.", e) diff --git a/bambu_octoeverywhere/bambumodels.py b/bambu_octoeverywhere/bambumodels.py index 9d4bfdb..d3abdae 100644 --- a/bambu_octoeverywhere/bambumodels.py +++ b/bambu_octoeverywhere/bambumodels.py @@ -83,12 +83,31 @@ def GetContinuousTimeRemainingSec(self) -> int: # Since there's a lot to consider to figure out if a print is running, this one function acts as common logic across the plugin. def IsPrinting(self, includePausedAsPrinting:bool) -> bool: - if self.gcode_state is None: + return BambuState.IsPrintingState(self.gcode_state, includePausedAsPrinting) + + + # Since there's a lot to consider to figure out if a print is running, this one function acts as common logic across the plugin. + def IsPrepareOrSlicing(self) -> bool: + return BambuState.IsPrepareOrSlicingState(self.gcode_state) + + + # We use this common method since "is this a printing state?" is complicated and we can to keep all of the logic common in the plugin + @staticmethod + def IsPrintingState(state:str, includePausedAsPrinting:bool) -> bool: + if state is None: return False - if self.gcode_state == "PAUSE" and includePausedAsPrinting: + if state == "PAUSE" and includePausedAsPrinting: return True # Do we need to consider some of the stg_cur states? - return self.gcode_state == "RUNNING" or self.gcode_state == "SLICING" or self.gcode_state == "PREPARE" + return state == "RUNNING" or BambuState.IsPrepareOrSlicingState(state) + + + # We use this common method to keep all of the logic common in the plugin + @staticmethod + def IsPrepareOrSlicingState(state:str) -> bool: + if state is None: + return False + return state == "SLICING" or state == "PREPARE" # This one function acts as common logic across the plugin. @@ -211,13 +230,13 @@ def OnUpdate(self, msg:dict) -> None: self.Cpu = BambuCPUs.Unknown # Now that we have info, map the printer type. - if self.Cpu is not BambuCPUs.Unknown and self.HardwareVersion is not None and self.ProjectName is not None: + if self.Cpu is not BambuCPUs.Unknown and self.HardwareVersion is not None: if self.Cpu is BambuCPUs.RV1126: if self.HardwareVersion == "AP05": self.PrinterName = BambuPrinters.X1C elif self.HardwareVersion == "AP02": self.PrinterName = BambuPrinters.X1E - if self.Cpu is BambuCPUs.ESP32: + if self.Cpu is BambuCPUs.ESP32 and self.ProjectName is not None: if self.HardwareVersion == "AP04": if self.ProjectName == "C11": self.PrinterName = BambuPrinters.P1P diff --git a/bambu_octoeverywhere/bambustatetranslater.py b/bambu_octoeverywhere/bambustatetranslater.py index b13ea3a..0261fda 100644 --- a/bambu_octoeverywhere/bambustatetranslater.py +++ b/bambu_octoeverywhere/bambustatetranslater.py @@ -43,6 +43,7 @@ def OnMqttMessage(self, msg:dict, bambuState:BambuState, isFirstFullSyncResponse # Here's a list of all states: https://github.com/greghesp/ha-bambulab/blob/e72e343acd3279c9bccba510f94bf0e291fe5aaa/custom_components/bambu_lab/pybambu/const.py#L83C1-L83C21 if self.LastState != bambuState.gcode_state: # We know the state changed. + self.Logger.debug(f"Bambu state change: {self.LastState} -> {bambuState.gcode_state}") if self.LastState is None: # If the last state is None, this is mostly likely the first time we've seen a state. # All we want to do here is update last state to the new state. @@ -52,7 +53,10 @@ def OnMqttMessage(self, msg:dict, bambuState:BambuState, isFirstFullSyncResponse if self.LastState == "PAUSE": self.BambuOnResume(bambuState) else: - self.BambuOnStart(bambuState) + # We know the state changed and the state is now a printing state. + # If the last state was also a printing state, we don't want to fire this, since we already did. + if BambuState.IsPrintingState(self.LastState, False) is False: + self.BambuOnStart(bambuState) # Check for the paused state elif bambuState.IsPaused(): # If the error is temporary, like a filament run out, the printer goes into a paused state @@ -81,7 +85,10 @@ def OnMqttMessage(self, msg:dict, bambuState:BambuState, isFirstFullSyncResponse # Percentage progress update printMsg = msg.get("print", None) if printMsg is not None and "mc_percent" in printMsg: - self.BambuOnPrintProgress(bambuState) + # On the X1, the progress doesn't get reset from the last print when the printer switches into prepare or slicing for the next print. + # So we will not send any progress updates in these states, until the state is "RUNNING" and the progress should reset to 0. + if bambuState.IsPrepareOrSlicing() is False: + self.BambuOnPrintProgress(bambuState) # Since bambu doesn't tell us a print duration, we need to figure out when it ends ourselves. # This is different from the state changes above, because if we are ever not printing for any reason, diff --git a/bambu_octoeverywhere/quickcam.py b/bambu_octoeverywhere/quickcam.py index cdc9f0d..c1ae39d 100644 --- a/bambu_octoeverywhere/quickcam.py +++ b/bambu_octoeverywhere/quickcam.py @@ -155,6 +155,7 @@ def _captureThread(self): # If the URL has an address, it's an X1 with LAN streaming. # If it's None, it's a P1, A1, or another printer with no RTSP. if rtspUrl is not None: + self.Logger.debug(f"Bambu RTSP URL is: `{rtspUrl}`") camImpl = QuickCam_RTSP(self.Logger) else: # Default to the websocket impl, since it's used on the most printers. @@ -367,7 +368,7 @@ def Connect(self, ipOrHostname:str, accessCode:str) -> None: "-hide_banner", "-y", "-loglevel", logLevel, - "-rtsp_transport", "tcp", + "-rtsp_transport", "udp", "-use_wallclock_as_timestamps", "1", "-i", f"rtsps://bblp:{accessCode}@{ipOrHostname}:322/streaming/live/1", "-filter:v", "fps=15", diff --git a/moonraker_octoeverywhere/uiinjector.py b/moonraker_octoeverywhere/uiinjector.py index ac20e58..32508db 100644 --- a/moonraker_octoeverywhere/uiinjector.py +++ b/moonraker_octoeverywhere/uiinjector.py @@ -223,7 +223,10 @@ def _UpdateExistingInjections(self, indexHtmlFilePath): self.Logger.info("Found existing ui tags but the hash didn't match, so we updated the hash.") return True, True except Exception as e: - Sentry.Exception("_InjectIntoHtml failed for "+indexHtmlFilePath, e) + if e is UnicodeDecodeError and "invalid continuation byte" in str(e): + self.Logger.warn("_InjectIntoHtml failed, the html file has invalid utf-8") + else: + Sentry.Exception("_InjectIntoHtml failed for "+indexHtmlFilePath, e) return False, False @@ -272,7 +275,10 @@ def _InjectIntoHtml(self, indexHtmlFilePath) -> bool: self.Logger.info("No existing ui tags found, so we added them") return True except Exception as e: - Sentry.Exception("_InjectIntoHtml failed for "+indexHtmlFilePath, e) + if e is UnicodeDecodeError and "invalid continuation byte" in str(e): + self.Logger.warn("Failed to inject UI helpers into html, the html file has invalid utf-8") + else: + Sentry.Exception("_InjectIntoHtml failed for "+indexHtmlFilePath, e) return False diff --git a/octoeverywhere/notificationshandler.py b/octoeverywhere/notificationshandler.py index 95f8065..a880cce 100644 --- a/octoeverywhere/notificationshandler.py +++ b/octoeverywhere/notificationshandler.py @@ -902,6 +902,8 @@ def GetNotificationSnapshot(self, snapshotResizeParams = None): # Note that in the case of an exception we don't overwrite the original snapshot buffer, so something can still be sent. if "name 'Image' is not defined" in str(e): self.Logger.info("Can't manipulate image because the Image rotation lib failed to import.") + if "cannot identify image file" in str(e): + self.Logger.info("Can't manipulate image because the Image lib can't figure out the image type.") else: Sentry.Exception("Failed to manipulate image for notifications", e) diff --git a/octoeverywhere/webcamhelper.py b/octoeverywhere/webcamhelper.py index 8576b5a..fd9fa21 100644 --- a/octoeverywhere/webcamhelper.py +++ b/octoeverywhere/webcamhelper.py @@ -378,23 +378,15 @@ def _GetSnapshotFromStream(self, url) -> OctoHttpRequest.Result: # Return a result. Return the full image buffer which will be used as the response body. self.Logger.debug("Successfully got image from stream URL. Size: %s, Format: %s", str(len(imageBuffer)), contentType) return OctoHttpRequest.Result(200, headers, url, True, fullBodyBuffer=imageBuffer) - except ConnectionError as e: - # We have a lot of telemetry indicating a read timeout can happen while trying to read from the stream - # in that case we should just get out of here. - if "Read timed out" in str(e): + except Exception as e: + if e is ConnectionError and "Read timed out" in str(e): self.Logger.debug("_GetSnapshotFromStream got a timeout while reading the stream.") - return None - else: - Sentry.Exception("Failed to get fallback snapshot due to ConnectionError", e) - except urllib3.exceptions.ProtocolError as e: - if "IncompleteRead" in str(e): + elif e is urllib3.exceptions.ProtocolError and "IncompleteRead" in str(e): self.Logger.debug("_GetSnapshotFromStream got a incomplete read while reading the stream.") - return None + elif e is urllib3.exceptions.ReadTimeoutError and "Read timed out" in str(e): + self.Logger.debug("_GetSnapshotFromStream got a read timeout while reading stream.") else: - Sentry.Exception("Failed to get fallback snapshot due to ProtocolError", e) - except Exception as e: - Sentry.Exception("Failed to get fallback snapshot.", e) - + Sentry.Exception("Failed to get fallback snapshot.", e) return None diff --git a/setup.py b/setup.py index 7c98b16..0ad7733 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this is also parsed by the moonraker module to pull the version, so the string and format must remain the same! -plugin_version = "3.2.3" +plugin_version = "3.2.4" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From f413eca6d5ebb4524f48b337d6869a565035d602 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Fri, 29 Mar 2024 15:05:26 -0700 Subject: [PATCH 069/328] Adding some test logic to try to figure out the strange random resume notification. --- moonraker_octoeverywhere/moonrakerclient.py | 1 + octoeverywhere/notificationshandler.py | 1 + octoeverywhere/sentry.py | 6 ++++++ setup.py | 2 +- 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/moonraker_octoeverywhere/moonrakerclient.py b/moonraker_octoeverywhere/moonrakerclient.py index 263638f..fbb3059 100644 --- a/moonraker_octoeverywhere/moonrakerclient.py +++ b/moonraker_octoeverywhere/moonrakerclient.py @@ -447,6 +447,7 @@ def _OnWsNonResponseMessage(self, msg:str): # so they don't trigger something too much lower too easily. elif state == "printing": if progressFloat_CanBeNone is None or progressFloat_CanBeNone > 0.01: + Sentry.Breadcrumb("Sending Resume Notification", stateContainerObj) self.MoonrakerCompat.OnPrintResumed() return elif state == "complete": diff --git a/octoeverywhere/notificationshandler.py b/octoeverywhere/notificationshandler.py index a880cce..a88312a 100644 --- a/octoeverywhere/notificationshandler.py +++ b/octoeverywhere/notificationshandler.py @@ -389,6 +389,7 @@ def OnPaused(self, fileName:str = None): # Fired when a print is resumed def OnResume(self, fileName:str = None): + Sentry.Breadcrumb("OnResume called.", {"filename":fileName}) if self._shouldIgnoreEvent(fileName): return self._updateCurrentFileName(fileName) diff --git a/octoeverywhere/sentry.py b/octoeverywhere/sentry.py index b5498d2..dd79c65 100644 --- a/octoeverywhere/sentry.py +++ b/octoeverywhere/sentry.py @@ -128,6 +128,12 @@ def _beforeSendFilter(event, hint): return event + # Adds a breadcrumb to the sentry log, which is helpful to figure out what happened before an exception. + @staticmethod + def Breadcrumb(msg:str, data:dict = None, level:str = "info", category:str = "breadcrumb"): + sentry_sdk.add_breadcrumb(message=msg, data=data, level=level, category=category) + + # Sends an error log to sentry. # This is useful for debugging things that shouldn't be happening. @staticmethod diff --git a/setup.py b/setup.py index 0ad7733..6f6e34d 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this is also parsed by the moonraker module to pull the version, so the string and format must remain the same! -plugin_version = "3.2.4" +plugin_version = "3.2.5" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 6c3e9caa6631cff1da0e9abf8645f7b307ea40c6 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Fri, 29 Mar 2024 17:34:08 -0700 Subject: [PATCH 070/328] Addnig more debugging. --- moonraker_octoeverywhere/moonrakerclient.py | 2 ++ setup.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/moonraker_octoeverywhere/moonrakerclient.py b/moonraker_octoeverywhere/moonrakerclient.py index fbb3059..595e3ca 100644 --- a/moonraker_octoeverywhere/moonrakerclient.py +++ b/moonraker_octoeverywhere/moonrakerclient.py @@ -827,6 +827,7 @@ def OnMoonrakerClientConnected(self): self._InitPrintStateForFreshConnect() # We are ready to process notifications! + Sentry.Breadcrumb("Moonraker client connected, print state restored, and we are ready to accept notifications.") self.IsReadyToProcessNotifications = True @@ -838,6 +839,7 @@ def KlippyDisconnectedOrShutdown(self): return # Set the flag to false again, since we can't send any more notifications until we are reconnected and re-synced. + Sentry.Breadcrumb("Moonraker disconnected. Stopping notification processing.") self.IsReadyToProcessNotifications = False # Only fire this error if we are tracking a print. diff --git a/setup.py b/setup.py index 6f6e34d..412ef9b 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this is also parsed by the moonraker module to pull the version, so the string and format must remain the same! -plugin_version = "3.2.5" +plugin_version = "3.2.6" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 6069e87adfe3a25f8f0672c9f01f135300f35b19 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sat, 30 Mar 2024 09:03:21 -0700 Subject: [PATCH 071/328] Adding a work around for the WS lib thread deadlock bug. --- bambu_octoeverywhere/bambuhost.py | 2 +- moonraker_octoeverywhere/moonrakerhost.py | 2 +- octoeverywhere/sentry.py | 50 ++++++++++++++++++++++- octoeverywhere/threaddebug.py | 15 +++++-- setup.py | 2 +- 5 files changed, 63 insertions(+), 8 deletions(-) diff --git a/bambu_octoeverywhere/bambuhost.py b/bambu_octoeverywhere/bambuhost.py index 2409dfa..0102867 100644 --- a/bambu_octoeverywhere/bambuhost.py +++ b/bambu_octoeverywhere/bambuhost.py @@ -71,7 +71,7 @@ def RunBlocking(self, configPath, localStorageDir, repoRoot, devConfig_CanBeNone # As soon as we have the plugin version, setup Sentry # Enabling profiling and no filtering, since we are the only PY in this process. - Sentry.Setup(pluginVersionStr, "bambu", devConfig_CanBeNone is not None, enableProfiling=True, filterExceptionsByPackage=False) + Sentry.Setup(pluginVersionStr, "bambu", devConfig_CanBeNone is not None, enableProfiling=True, filterExceptionsByPackage=False, restartOnCantCreateThreadBug=True) # Before the first time setup, we must also init the Secrets class and do the migration for the printer id and private key, if needed. self.Secrets = Secrets(self.Logger, localStorageDir) diff --git a/moonraker_octoeverywhere/moonrakerhost.py b/moonraker_octoeverywhere/moonrakerhost.py index 12007da..0cea94b 100644 --- a/moonraker_octoeverywhere/moonrakerhost.py +++ b/moonraker_octoeverywhere/moonrakerhost.py @@ -87,7 +87,7 @@ def RunBlocking(self, klipperConfigDir, isCompanionMode, localStorageDir, servic # As soon as we have the plugin version, setup Sentry # Enabling profiling and no filtering, since we are the only PY in this process. - Sentry.Setup(pluginVersionStr, "klipper", devConfig_CanBeNone is not None, enableProfiling=True, filterExceptionsByPackage=False) + Sentry.Setup(pluginVersionStr, "klipper", devConfig_CanBeNone is not None, enableProfiling=True, filterExceptionsByPackage=False, restartOnCantCreateThreadBug=True) # This logic only works if running locally. if not isCompanionMode: diff --git a/octoeverywhere/sentry.py b/octoeverywhere/sentry.py index dd79c65..c0a49e2 100644 --- a/octoeverywhere/sentry.py +++ b/octoeverywhere/sentry.py @@ -1,12 +1,15 @@ +import os import logging import time import traceback import sentry_sdk +from sentry_sdk import Hub from sentry_sdk.integrations.logging import LoggingIntegration from sentry_sdk.integrations.threading import ThreadingIntegration from .exceptions import NoSentryReportException +from .threaddebug import ThreadDebug # A helper class to handle Sentry logic. class Sentry: @@ -20,6 +23,7 @@ class Sentry: FilterExceptionsByPackage:bool = False LastErrorReport:float = time.time() LastErrorCount:int = 0 + RestartProcessOnCantCreateThreadBug = False # This will be called as soon as possible when the process starts to capture the logger, so it's ready for use. @@ -31,10 +35,11 @@ def SetLogger(logger:logging.Logger): # This actually setups sentry. # It's only called after the plugin version is known, and thus it might be a little into the process lifetime. @staticmethod - def Setup(versionString:str, distType:str, isDevMode:bool = False, enableProfiling:bool = False, filterExceptionsByPackage:bool = False): + def Setup(versionString:str, distType:str, isDevMode:bool = False, enableProfiling:bool = False, filterExceptionsByPackage:bool = False, restartOnCantCreateThreadBug:bool = False): # Set the dev mode flag. Sentry.IsDevMode = isDevMode Sentry.FilterExceptionsByPackage = filterExceptionsByPackage + Sentry.RestartProcessOnCantCreateThreadBug = restartOnCantCreateThreadBug # Only setup sentry if we aren't in dev mode. if Sentry.IsDevMode is False: @@ -172,6 +177,11 @@ def _handleException(msg:str, exception:Exception, sendException:bool, extras:di if Sentry._Logger is None: return + # We have special logic to handle a bug were we can't create new threads due to a deadlock + # in our websocket lib. This logic will do that, if it returns true, the Exception has handled. + if Sentry._HandleCantCreateThreadException(Sentry._Logger, exception): + return + tb = traceback.format_exc() exceptionClassType = "unknown_type" if exception is not None: @@ -191,3 +201,41 @@ def _handleException(msg:str, exception:Exception, sendException:bool, extras:di for key, value in extras.items(): scope.set_extra(key, value) sentry_sdk.capture_exception(exception) + + + # If the exception is that we can't start new thread, this logs it, and then restarts if needed. + # Returns of the exception was handled. + _IsHandlingCantCreateThreadException = False + @staticmethod + def _HandleCantCreateThreadException(logger:logging.Logger, e:Exception) -> bool: + # Filter the exception + if e is not RuntimeError or "can't start new thread" not in str(e): + return False + + # If we can't restart, return false, and the normal exception handling will occur. + if Sentry.RestartProcessOnCantCreateThreadBug is False: + return False + + # If we are already handling this, return False to prevent a loop. + # We return false so the exception we are reporting will be handled in the normal way. + if Sentry._IsHandlingCantCreateThreadException: + return False + Sentry._IsHandlingCantCreateThreadException = True + + # Log the error + ThreadDebug.DoThreadDumpLogout(logger, True) + logger.error("~~~~~~~~~ Process Restarting Due To Threading Bug ~~~~~~~~~~~~") + Sentry.Exception("Can't start new thread - restarting the process.", e) + + # Flush Sentry + # Once this is called, Sentry is shutdown, so we must restart. + try: + client = Hub.current.client + if client is not None: + client.close(timeout=5.0) + except Exception: + pass + + # Restart the process - We must use this function to actually force the process to exit + # The systemd handler will restart us. + os.abort() diff --git a/octoeverywhere/threaddebug.py b/octoeverywhere/threaddebug.py index cd7b7a4..e82c0b6 100644 --- a/octoeverywhere/threaddebug.py +++ b/octoeverywhere/threaddebug.py @@ -1,20 +1,20 @@ import threading +import logging import time import sys import traceback + class ThreadDebug: def Start(self, logger, delaySec): try: th = threading.Thread(target=self.threadWorker, args=(logger, delaySec)) - # pylint: disable=deprecated-method - # This is deprecated in PY3.10 - th.setDaemon(True) th.start() except Exception as e: logger.error("Failed to start Thread Debug Thread: "+str(e)) + def threadWorker(self, logger, delaySec): while True: try: @@ -24,9 +24,11 @@ def threadWorker(self, logger, delaySec): logger.error("Exception in ThreadDebug : "+str(e)) time.sleep(delaySec) + @staticmethod - def DoThreadDumpLogout(logger): + def DoThreadDumpLogout(logger:logging.Logger, addSentryBreadcrumb:bool = False): try: + sentryData = {} logger.info("ThreadDump - Starting Thread Dump") # pylint: disable=protected-access for threadId, stack in sys._current_frames().items(): @@ -40,5 +42,10 @@ def DoThreadDumpLogout(logger): else: trace += ", "+filename+":"+name logger.info("ThreadDump- Id: "+str(threadId) + " -> "+str(trace)) + sentryData[str(threadId)] = str(trace) + + # Dump the sentry data if desired. + if addSentryBreadcrumb: + Sentry.Breadcrumb("ThreadDump", sentryData) except Exception as e: logger.error("Exception in ThreadDebug : "+str(e)) diff --git a/setup.py b/setup.py index 412ef9b..debee77 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this is also parsed by the moonraker module to pull the version, so the string and format must remain the same! -plugin_version = "3.2.6" +plugin_version = "3.2.7" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 0999a21b197bba3130070e8892e03d01aa233e7a Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sat, 30 Mar 2024 09:06:18 -0700 Subject: [PATCH 072/328] Fixing a small bug. --- octoeverywhere/threaddebug.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/octoeverywhere/threaddebug.py b/octoeverywhere/threaddebug.py index e82c0b6..f102cb6 100644 --- a/octoeverywhere/threaddebug.py +++ b/octoeverywhere/threaddebug.py @@ -26,9 +26,8 @@ def threadWorker(self, logger, delaySec): @staticmethod - def DoThreadDumpLogout(logger:logging.Logger, addSentryBreadcrumb:bool = False): + def DoThreadDumpLogout(logger:logging.Logger): try: - sentryData = {} logger.info("ThreadDump - Starting Thread Dump") # pylint: disable=protected-access for threadId, stack in sys._current_frames().items(): @@ -42,10 +41,5 @@ def DoThreadDumpLogout(logger:logging.Logger, addSentryBreadcrumb:bool = False): else: trace += ", "+filename+":"+name logger.info("ThreadDump- Id: "+str(threadId) + " -> "+str(trace)) - sentryData[str(threadId)] = str(trace) - - # Dump the sentry data if desired. - if addSentryBreadcrumb: - Sentry.Breadcrumb("ThreadDump", sentryData) except Exception as e: logger.error("Exception in ThreadDebug : "+str(e)) From 15f45c77d8fe5d735974133dc927cc0428d3f3f5 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sat, 30 Mar 2024 09:11:19 -0700 Subject: [PATCH 073/328] Lint fix. --- octoeverywhere/sentry.py | 1 + 1 file changed, 1 insertion(+) diff --git a/octoeverywhere/sentry.py b/octoeverywhere/sentry.py index c0a49e2..1cc9fd9 100644 --- a/octoeverywhere/sentry.py +++ b/octoeverywhere/sentry.py @@ -207,6 +207,7 @@ def _handleException(msg:str, exception:Exception, sendException:bool, extras:di # Returns of the exception was handled. _IsHandlingCantCreateThreadException = False @staticmethod + # pylint: inconsistent-return-statements def _HandleCantCreateThreadException(logger:logging.Logger, e:Exception) -> bool: # Filter the exception if e is not RuntimeError or "can't start new thread" not in str(e): From d7bd4d9df209e62b171b3590e84f29bc65222e15 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sun, 31 Mar 2024 11:39:45 -0700 Subject: [PATCH 074/328] Lint fix. --- octoeverywhere/sentry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octoeverywhere/sentry.py b/octoeverywhere/sentry.py index 1cc9fd9..58a6871 100644 --- a/octoeverywhere/sentry.py +++ b/octoeverywhere/sentry.py @@ -206,8 +206,8 @@ def _handleException(msg:str, exception:Exception, sendException:bool, extras:di # If the exception is that we can't start new thread, this logs it, and then restarts if needed. # Returns of the exception was handled. _IsHandlingCantCreateThreadException = False + # pylint: disable=inconsistent-return-statements @staticmethod - # pylint: inconsistent-return-statements def _HandleCantCreateThreadException(logger:logging.Logger, e:Exception) -> bool: # Filter the exception if e is not RuntimeError or "can't start new thread" not in str(e): From 70eeaabc2456d94503c12c8468d4d0537cd3819c Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Tue, 2 Apr 2024 20:35:39 -0700 Subject: [PATCH 075/328] Adding logic to better shutdown ffmpeg for the x1. --- .github/workflows/pylint.yml | 2 +- bambu_octoeverywhere/quickcam.py | 28 ++++++++++++++++++++++++++-- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index c551a1b..1491014 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -8,7 +8,7 @@ jobs: strategy: matrix: # The sonic pad runs 3.7, so it's important to keep it here to make sure all of our required dependencies work - python-version: ["3.7", "3.8", "3.9", "3.10"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} diff --git a/bambu_octoeverywhere/quickcam.py b/bambu_octoeverywhere/quickcam.py index c1ae39d..f2a8d7e 100644 --- a/bambu_octoeverywhere/quickcam.py +++ b/bambu_octoeverywhere/quickcam.py @@ -7,6 +7,7 @@ import socket import time import os +import signal from octoeverywhere.sentry import Sentry @@ -407,7 +408,7 @@ def GetImage(self) -> bytearray: if self.Process.returncode is not None or (time.time() - self.TimeSinceLastImg) > QuickCam_RTSP.c_ReadTimeoutSec: if self.StdErrBuffer is None or len(self.StdErrBuffer) == 0: self.StdErrBuffer = "" - raise Exception(f"Ffmpeg read timeout. StdError: {self.StdErrBuffer}") + raise Exception(f"Ffmpeg read timeout. ffmpeg output:\n{self.StdErrBuffer}") # If we get an empty buffer, we just need to wait for more. if buffer is None or len(buffer) == 0: @@ -547,18 +548,41 @@ def __exit__(self, t, v, tb): # Killing the process will cause the error reader thread to exit. self.ErrorReaderThreadRunning = False + # First, we want to try to gracefully kill ffmpeg. That way it has time to tell the rtsp server it's + # going away and clean up. + try: + if self.Process is not None: + # Send sig int to emulate a ctl+c + self.Process.send_signal(signal.SIGINT) + + # Use communicate which will wait for the process to end and read it's final output. + # We also try to issue the q terminal command to exit, just incase the ffmpeg needs it. + # Give ffmpeg a good amount of time to exit, so ideally it gracefully exits. (usually this is really quick) + _, stderr =self.Process.communicate("q\r\n".encode("utf-8"), timeout=10.0) + + # Report what happened. + # For some reason communicate will eat the output instead of it being sent to our reader above, so we just print it here as well. + if stderr is None: + stderr = b"" + stderr = stderr.decode("utf-8") + self.Logger.debug(f"ffmpeg gracefully killed. Remaining ffmpeg output:\n{stderr}") + except Exception as e: + self.Logger.warn(f"Exception when trying to gracefully kill ffmpeg. {e}") + # Close in the opposite order they were opened. try: if self.PipeSelect is not None: self.PipeSelect.close() except Exception: pass - # Tell the process to be killed + + # Ensure the process is killed try: if self.Process is not None: self.Process.kill() except Exception: pass + # And then call exit to cleanup all of the pipes and process handles. try: if self.Process is not None: From 912ba5f9d217743c5a1edfbf54e2217571bc8d06 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Wed, 24 Apr 2024 22:00:43 -0700 Subject: [PATCH 076/328] Upgrading the OctoEverywhere client to server protool --- .github/workflows/pylint.yml | 1 + octoeverywhere/Proto/DataCompression.py | 1 - octoeverywhere/Proto/HandshakeAck.py | 117 +++++---- octoeverywhere/Proto/HandshakeSyn.py | 211 +++++++++------- octoeverywhere/Proto/HttpHeader.py | 49 ++-- octoeverywhere/Proto/HttpInitialContext.py | 109 +++++---- octoeverywhere/Proto/MessageContext.py | 1 - octoeverywhere/Proto/MessagePriority.py | 1 - octoeverywhere/Proto/OctoNotification.py | 103 ++++---- octoeverywhere/Proto/OctoNotificationTypes.py | 1 - octoeverywhere/Proto/OctoStreamMessage.py | 49 ++-- octoeverywhere/Proto/OctoSummon.py | 47 ++-- octoeverywhere/Proto/OeAuthAllowed.py | 1 - octoeverywhere/Proto/OsType.py | 1 - octoeverywhere/Proto/PathTypes.py | 1 - octoeverywhere/Proto/ServerHost.py | 1 - octoeverywhere/Proto/SummonMethods.py | 1 - octoeverywhere/Proto/WebSocketDataTypes.py | 1 - octoeverywhere/Proto/WebStreamMsg.py | 225 +++++++++++------- requirements.txt | 2 +- setup.py | 2 +- 21 files changed, 542 insertions(+), 383 deletions(-) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 1491014..85520d0 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -8,6 +8,7 @@ jobs: strategy: matrix: # The sonic pad runs 3.7, so it's important to keep it here to make sure all of our required dependencies work + # As of 4-23-2024 OctoPrint doesn't support 3.12, so we can't test it because it will fail the pip install. python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v2 diff --git a/octoeverywhere/Proto/DataCompression.py b/octoeverywhere/Proto/DataCompression.py index 26a5876..283c213 100644 --- a/octoeverywhere/Proto/DataCompression.py +++ b/octoeverywhere/Proto/DataCompression.py @@ -6,4 +6,3 @@ class DataCompression(object): None_ = 0 Brotli = 1 Zlib = 2 - diff --git a/octoeverywhere/Proto/HandshakeAck.py b/octoeverywhere/Proto/HandshakeAck.py index cd55315..f909da7 100644 --- a/octoeverywhere/Proto/HandshakeAck.py +++ b/octoeverywhere/Proto/HandshakeAck.py @@ -3,11 +3,13 @@ # namespace: Proto import octoflatbuffers +from typing import Any +from typing import Optional class HandshakeAck(object): __slots__ = ['_tab'] @classmethod - def GetRootAs(cls, buf, offset=0): + def GetRootAs(cls, buf, offset: int = 0): n = octoflatbuffers.encode.Get(octoflatbuffers.packer.uoffset, buf, offset) x = HandshakeAck() x.Init(buf, n + offset) @@ -18,7 +20,7 @@ def GetRootAsHandshakeAck(cls, buf, offset=0): """This method is deprecated. Please switch to GetRootAs.""" return cls.GetRootAs(buf, offset) # HandshakeAck - def Init(self, buf, pos): + def Init(self, buf: bytes, pos: int): self._tab = octoflatbuffers.table.Table(buf, pos) # HandshakeAck @@ -29,7 +31,7 @@ def Accepted(self): return False # HandshakeAck - def ConnectedAccounts(self, j): + def ConnectedAccounts(self, j: int): o = octoflatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) if o != 0: a = self._tab.Vector(o) @@ -37,19 +39,19 @@ def ConnectedAccounts(self, j): return "" # HandshakeAck - def ConnectedAccountsLength(self): + def ConnectedAccountsLength(self) -> int: o = octoflatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) if o != 0: return self._tab.VectorLen(o) return 0 # HandshakeAck - def ConnectedAccountsIsNone(self): + def ConnectedAccountsIsNone(self) -> bool: o = octoflatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) return o == 0 # HandshakeAck - def Error(self): + def Error(self) -> Optional[str]: o = octoflatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8)) if o != 0: return self._tab.String(o + self._tab.Pos) @@ -70,56 +72,75 @@ def RequiresPluginUpdate(self): return False # HandshakeAck - def Octokey(self): + def Octokey(self) -> Optional[str]: o = octoflatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14)) if o != 0: return self._tab.String(o + self._tab.Pos) return None # HandshakeAck - def RsaChallengeResult(self): + def RsaChallengeResult(self) -> Optional[str]: o = octoflatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(16)) if o != 0: return self._tab.String(o + self._tab.Pos) return None -def Start(builder): builder.StartObject(7) -def HandshakeAckStart(builder): - """This method is deprecated. Please switch to Start.""" - return Start(builder) -def AddAccepted(builder, accepted): builder.PrependBoolSlot(0, accepted, 0) -def HandshakeAckAddAccepted(builder, accepted): - """This method is deprecated. Please switch to AddAccepted.""" - return AddAccepted(builder, accepted) -def AddConnectedAccounts(builder, connectedAccounts): builder.PrependUOffsetTRelativeSlot(1, octoflatbuffers.number_types.UOffsetTFlags.py_type(connectedAccounts), 0) -def HandshakeAckAddConnectedAccounts(builder, connectedAccounts): - """This method is deprecated. Please switch to AddConnectedAccounts.""" - return AddConnectedAccounts(builder, connectedAccounts) -def StartConnectedAccountsVector(builder, numElems): return builder.StartVector(4, numElems, 4) -def HandshakeAckStartConnectedAccountsVector(builder, numElems): - """This method is deprecated. Please switch to Start.""" - return StartConnectedAccountsVector(builder, numElems) -def AddError(builder, error): builder.PrependUOffsetTRelativeSlot(2, octoflatbuffers.number_types.UOffsetTFlags.py_type(error), 0) -def HandshakeAckAddError(builder, error): - """This method is deprecated. Please switch to AddError.""" - return AddError(builder, error) -def AddBackoffSeconds(builder, backoffSeconds): builder.PrependUint64Slot(3, backoffSeconds, 0) -def HandshakeAckAddBackoffSeconds(builder, backoffSeconds): - """This method is deprecated. Please switch to AddBackoffSeconds.""" - return AddBackoffSeconds(builder, backoffSeconds) -def AddRequiresPluginUpdate(builder, requiresPluginUpdate): builder.PrependBoolSlot(4, requiresPluginUpdate, 0) -def HandshakeAckAddRequiresPluginUpdate(builder, requiresPluginUpdate): - """This method is deprecated. Please switch to AddRequiresPluginUpdate.""" - return AddRequiresPluginUpdate(builder, requiresPluginUpdate) -def AddOctokey(builder, octokey): builder.PrependUOffsetTRelativeSlot(5, octoflatbuffers.number_types.UOffsetTFlags.py_type(octokey), 0) -def HandshakeAckAddOctokey(builder, octokey): - """This method is deprecated. Please switch to AddOctokey.""" - return AddOctokey(builder, octokey) -def AddRsaChallengeResult(builder, rsaChallengeResult): builder.PrependUOffsetTRelativeSlot(6, octoflatbuffers.number_types.UOffsetTFlags.py_type(rsaChallengeResult), 0) -def HandshakeAckAddRsaChallengeResult(builder, rsaChallengeResult): - """This method is deprecated. Please switch to AddRsaChallengeResult.""" - return AddRsaChallengeResult(builder, rsaChallengeResult) -def End(builder): return builder.EndObject() -def HandshakeAckEnd(builder): - """This method is deprecated. Please switch to End.""" - return End(builder) \ No newline at end of file +def HandshakeAckStart(builder: octoflatbuffers.Builder): + builder.StartObject(7) + +def Start(builder: octoflatbuffers.Builder): + HandshakeAckStart(builder) + +def HandshakeAckAddAccepted(builder: octoflatbuffers.Builder, accepted: bool): + builder.PrependBoolSlot(0, accepted, 0) + +def AddAccepted(builder: octoflatbuffers.Builder, accepted: bool): + HandshakeAckAddAccepted(builder, accepted) + +def HandshakeAckAddConnectedAccounts(builder: octoflatbuffers.Builder, connectedAccounts: int): + builder.PrependUOffsetTRelativeSlot(1, octoflatbuffers.number_types.UOffsetTFlags.py_type(connectedAccounts), 0) + +def AddConnectedAccounts(builder: octoflatbuffers.Builder, connectedAccounts: int): + HandshakeAckAddConnectedAccounts(builder, connectedAccounts) + +def HandshakeAckStartConnectedAccountsVector(builder, numElems: int) -> int: + return builder.StartVector(4, numElems, 4) + +def StartConnectedAccountsVector(builder, numElems: int) -> int: + return HandshakeAckStartConnectedAccountsVector(builder, numElems) + +def HandshakeAckAddError(builder: octoflatbuffers.Builder, error: int): + builder.PrependUOffsetTRelativeSlot(2, octoflatbuffers.number_types.UOffsetTFlags.py_type(error), 0) + +def AddError(builder: octoflatbuffers.Builder, error: int): + HandshakeAckAddError(builder, error) + +def HandshakeAckAddBackoffSeconds(builder: octoflatbuffers.Builder, backoffSeconds: int): + builder.PrependUint64Slot(3, backoffSeconds, 0) + +def AddBackoffSeconds(builder: octoflatbuffers.Builder, backoffSeconds: int): + HandshakeAckAddBackoffSeconds(builder, backoffSeconds) + +def HandshakeAckAddRequiresPluginUpdate(builder: octoflatbuffers.Builder, requiresPluginUpdate: bool): + builder.PrependBoolSlot(4, requiresPluginUpdate, 0) + +def AddRequiresPluginUpdate(builder: octoflatbuffers.Builder, requiresPluginUpdate: bool): + HandshakeAckAddRequiresPluginUpdate(builder, requiresPluginUpdate) + +def HandshakeAckAddOctokey(builder: octoflatbuffers.Builder, octokey: int): + builder.PrependUOffsetTRelativeSlot(5, octoflatbuffers.number_types.UOffsetTFlags.py_type(octokey), 0) + +def AddOctokey(builder: octoflatbuffers.Builder, octokey: int): + HandshakeAckAddOctokey(builder, octokey) + +def HandshakeAckAddRsaChallengeResult(builder: octoflatbuffers.Builder, rsaChallengeResult: int): + builder.PrependUOffsetTRelativeSlot(6, octoflatbuffers.number_types.UOffsetTFlags.py_type(rsaChallengeResult), 0) + +def AddRsaChallengeResult(builder: octoflatbuffers.Builder, rsaChallengeResult: int): + HandshakeAckAddRsaChallengeResult(builder, rsaChallengeResult) + +def HandshakeAckEnd(builder: octoflatbuffers.Builder) -> int: + return builder.EndObject() + +def End(builder: octoflatbuffers.Builder) -> int: + return HandshakeAckEnd(builder) diff --git a/octoeverywhere/Proto/HandshakeSyn.py b/octoeverywhere/Proto/HandshakeSyn.py index de13355..4dd19e6 100644 --- a/octoeverywhere/Proto/HandshakeSyn.py +++ b/octoeverywhere/Proto/HandshakeSyn.py @@ -3,11 +3,13 @@ # namespace: Proto import octoflatbuffers +from typing import Any +from typing import Optional class HandshakeSyn(object): __slots__ = ['_tab'] @classmethod - def GetRootAs(cls, buf, offset=0): + def GetRootAs(cls, buf, offset: int = 0): n = octoflatbuffers.encode.Get(octoflatbuffers.packer.uoffset, buf, offset) x = HandshakeSyn() x.Init(buf, n + offset) @@ -18,11 +20,11 @@ def GetRootAsHandshakeSyn(cls, buf, offset=0): """This method is deprecated. Please switch to GetRootAs.""" return cls.GetRootAs(buf, offset) # HandshakeSyn - def Init(self, buf, pos): + def Init(self, buf: bytes, pos: int): self._tab = octoflatbuffers.table.Table(buf, pos) # HandshakeSyn - def PrinterId(self): + def PrinterId(self) -> Optional[str]: o = octoflatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) if o != 0: return self._tab.String(o + self._tab.Pos) @@ -36,14 +38,14 @@ def IsPrimaryConnection(self): return False # HandshakeSyn - def PluginVersion(self): + def PluginVersion(self) -> Optional[str]: o = octoflatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8)) if o != 0: return self._tab.String(o + self._tab.Pos) return None # HandshakeSyn - def LocalDeviceIp(self): + def LocalDeviceIp(self) -> Optional[str]: o = octoflatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10)) if o != 0: return self._tab.String(o + self._tab.Pos) @@ -57,14 +59,14 @@ def LocalHttpProxyPort(self): return 0 # HandshakeSyn - def Key(self): + def Key(self) -> Optional[str]: o = octoflatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14)) if o != 0: return self._tab.String(o + self._tab.Pos) return None # HandshakeSyn - def RsaChallenge(self, j): + def RsaChallenge(self, j: int): o = octoflatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(16)) if o != 0: a = self._tab.Vector(o) @@ -86,14 +88,14 @@ def RsaChallengeAsByteArray(self): return 0 # HandshakeSyn - def RsaChallengeLength(self): + def RsaChallengeLength(self) -> int: o = octoflatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(16)) if o != 0: return self._tab.VectorLen(o) return 0 # HandshakeSyn - def RsaChallengeIsNone(self): + def RsaChallengeIsNone(self) -> bool: o = octoflatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(16)) return o == 0 @@ -126,7 +128,7 @@ def WebcamFlipRotate90(self): return False # HandshakeSyn - def PrivateKey(self): + def PrivateKey(self) -> Optional[str]: o = octoflatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(26)) if o != 0: return self._tab.String(o + self._tab.Pos) @@ -160,79 +162,116 @@ def OsType(self): return self._tab.Get(octoflatbuffers.number_types.Int8Flags, o + self._tab.Pos) return 0 -def Start(builder): builder.StartObject(16) -def HandshakeSynStart(builder): - """This method is deprecated. Please switch to Start.""" - return Start(builder) -def AddPrinterId(builder, printerId): builder.PrependUOffsetTRelativeSlot(0, octoflatbuffers.number_types.UOffsetTFlags.py_type(printerId), 0) -def HandshakeSynAddPrinterId(builder, printerId): - """This method is deprecated. Please switch to AddPrinterId.""" - return AddPrinterId(builder, printerId) -def AddIsPrimaryConnection(builder, isPrimaryConnection): builder.PrependBoolSlot(1, isPrimaryConnection, 0) -def HandshakeSynAddIsPrimaryConnection(builder, isPrimaryConnection): - """This method is deprecated. Please switch to AddIsPrimaryConnection.""" - return AddIsPrimaryConnection(builder, isPrimaryConnection) -def AddPluginVersion(builder, pluginVersion): builder.PrependUOffsetTRelativeSlot(2, octoflatbuffers.number_types.UOffsetTFlags.py_type(pluginVersion), 0) -def HandshakeSynAddPluginVersion(builder, pluginVersion): - """This method is deprecated. Please switch to AddPluginVersion.""" - return AddPluginVersion(builder, pluginVersion) -def AddLocalDeviceIp(builder, localDeviceIp): builder.PrependUOffsetTRelativeSlot(3, octoflatbuffers.number_types.UOffsetTFlags.py_type(localDeviceIp), 0) -def HandshakeSynAddLocalDeviceIp(builder, localDeviceIp): - """This method is deprecated. Please switch to AddLocalDeviceIp.""" - return AddLocalDeviceIp(builder, localDeviceIp) -def AddLocalHttpProxyPort(builder, localHttpProxyPort): builder.PrependUint32Slot(4, localHttpProxyPort, 0) -def HandshakeSynAddLocalHttpProxyPort(builder, localHttpProxyPort): - """This method is deprecated. Please switch to AddLocalHttpProxyPort.""" - return AddLocalHttpProxyPort(builder, localHttpProxyPort) -def AddKey(builder, key): builder.PrependUOffsetTRelativeSlot(5, octoflatbuffers.number_types.UOffsetTFlags.py_type(key), 0) -def HandshakeSynAddKey(builder, key): - """This method is deprecated. Please switch to AddKey.""" - return AddKey(builder, key) -def AddRsaChallenge(builder, rsaChallenge): builder.PrependUOffsetTRelativeSlot(6, octoflatbuffers.number_types.UOffsetTFlags.py_type(rsaChallenge), 0) -def HandshakeSynAddRsaChallenge(builder, rsaChallenge): - """This method is deprecated. Please switch to AddRsaChallenge.""" - return AddRsaChallenge(builder, rsaChallenge) -def StartRsaChallengeVector(builder, numElems): return builder.StartVector(1, numElems, 1) -def HandshakeSynStartRsaChallengeVector(builder, numElems): - """This method is deprecated. Please switch to Start.""" - return StartRsaChallengeVector(builder, numElems) -def AddRasChallengeVersion(builder, rasChallengeVersion): builder.PrependInt8Slot(7, rasChallengeVersion, 0) -def HandshakeSynAddRasChallengeVersion(builder, rasChallengeVersion): - """This method is deprecated. Please switch to AddRasChallengeVersion.""" - return AddRasChallengeVersion(builder, rasChallengeVersion) -def AddWebcamFlipH(builder, webcamFlipH): builder.PrependBoolSlot(8, webcamFlipH, 0) -def HandshakeSynAddWebcamFlipH(builder, webcamFlipH): - """This method is deprecated. Please switch to AddWebcamFlipH.""" - return AddWebcamFlipH(builder, webcamFlipH) -def AddWebcamFlipV(builder, webcamFlipV): builder.PrependBoolSlot(9, webcamFlipV, 0) -def HandshakeSynAddWebcamFlipV(builder, webcamFlipV): - """This method is deprecated. Please switch to AddWebcamFlipV.""" - return AddWebcamFlipV(builder, webcamFlipV) -def AddWebcamFlipRotate90(builder, webcamFlipRotate90): builder.PrependBoolSlot(10, webcamFlipRotate90, 0) -def HandshakeSynAddWebcamFlipRotate90(builder, webcamFlipRotate90): - """This method is deprecated. Please switch to AddWebcamFlipRotate90.""" - return AddWebcamFlipRotate90(builder, webcamFlipRotate90) -def AddPrivateKey(builder, privateKey): builder.PrependUOffsetTRelativeSlot(11, octoflatbuffers.number_types.UOffsetTFlags.py_type(privateKey), 0) -def HandshakeSynAddPrivateKey(builder, privateKey): - """This method is deprecated. Please switch to AddPrivateKey.""" - return AddPrivateKey(builder, privateKey) -def AddSummonMethod(builder, summonMethod): builder.PrependInt8Slot(12, summonMethod, 1) -def HandshakeSynAddSummonMethod(builder, summonMethod): - """This method is deprecated. Please switch to AddSummonMethod.""" - return AddSummonMethod(builder, summonMethod) -def AddServerHost(builder, serverHost): builder.PrependInt8Slot(13, serverHost, 0) -def HandshakeSynAddServerHost(builder, serverHost): - """This method is deprecated. Please switch to AddServerHost.""" - return AddServerHost(builder, serverHost) -def AddIsCompanion(builder, isCompanion): builder.PrependBoolSlot(14, isCompanion, 0) -def HandshakeSynAddIsCompanion(builder, isCompanion): - """This method is deprecated. Please switch to AddIsCompanion.""" - return AddIsCompanion(builder, isCompanion) -def AddOsType(builder, osType): builder.PrependInt8Slot(15, osType, 0) -def HandshakeSynAddOsType(builder, osType): - """This method is deprecated. Please switch to AddOsType.""" - return AddOsType(builder, osType) -def End(builder): return builder.EndObject() -def HandshakeSynEnd(builder): - """This method is deprecated. Please switch to End.""" - return End(builder) \ No newline at end of file +def HandshakeSynStart(builder: octoflatbuffers.Builder): + builder.StartObject(16) + +def Start(builder: octoflatbuffers.Builder): + HandshakeSynStart(builder) + +def HandshakeSynAddPrinterId(builder: octoflatbuffers.Builder, printerId: int): + builder.PrependUOffsetTRelativeSlot(0, octoflatbuffers.number_types.UOffsetTFlags.py_type(printerId), 0) + +def AddPrinterId(builder: octoflatbuffers.Builder, printerId: int): + HandshakeSynAddPrinterId(builder, printerId) + +def HandshakeSynAddIsPrimaryConnection(builder: octoflatbuffers.Builder, isPrimaryConnection: bool): + builder.PrependBoolSlot(1, isPrimaryConnection, 0) + +def AddIsPrimaryConnection(builder: octoflatbuffers.Builder, isPrimaryConnection: bool): + HandshakeSynAddIsPrimaryConnection(builder, isPrimaryConnection) + +def HandshakeSynAddPluginVersion(builder: octoflatbuffers.Builder, pluginVersion: int): + builder.PrependUOffsetTRelativeSlot(2, octoflatbuffers.number_types.UOffsetTFlags.py_type(pluginVersion), 0) + +def AddPluginVersion(builder: octoflatbuffers.Builder, pluginVersion: int): + HandshakeSynAddPluginVersion(builder, pluginVersion) + +def HandshakeSynAddLocalDeviceIp(builder: octoflatbuffers.Builder, localDeviceIp: int): + builder.PrependUOffsetTRelativeSlot(3, octoflatbuffers.number_types.UOffsetTFlags.py_type(localDeviceIp), 0) + +def AddLocalDeviceIp(builder: octoflatbuffers.Builder, localDeviceIp: int): + HandshakeSynAddLocalDeviceIp(builder, localDeviceIp) + +def HandshakeSynAddLocalHttpProxyPort(builder: octoflatbuffers.Builder, localHttpProxyPort: int): + builder.PrependUint32Slot(4, localHttpProxyPort, 0) + +def AddLocalHttpProxyPort(builder: octoflatbuffers.Builder, localHttpProxyPort: int): + HandshakeSynAddLocalHttpProxyPort(builder, localHttpProxyPort) + +def HandshakeSynAddKey(builder: octoflatbuffers.Builder, key: int): + builder.PrependUOffsetTRelativeSlot(5, octoflatbuffers.number_types.UOffsetTFlags.py_type(key), 0) + +def AddKey(builder: octoflatbuffers.Builder, key: int): + HandshakeSynAddKey(builder, key) + +def HandshakeSynAddRsaChallenge(builder: octoflatbuffers.Builder, rsaChallenge: int): + builder.PrependUOffsetTRelativeSlot(6, octoflatbuffers.number_types.UOffsetTFlags.py_type(rsaChallenge), 0) + +def AddRsaChallenge(builder: octoflatbuffers.Builder, rsaChallenge: int): + HandshakeSynAddRsaChallenge(builder, rsaChallenge) + +def HandshakeSynStartRsaChallengeVector(builder, numElems: int) -> int: + return builder.StartVector(1, numElems, 1) + +def StartRsaChallengeVector(builder, numElems: int) -> int: + return HandshakeSynStartRsaChallengeVector(builder, numElems) + +def HandshakeSynAddRasChallengeVersion(builder: octoflatbuffers.Builder, rasChallengeVersion: int): + builder.PrependInt8Slot(7, rasChallengeVersion, 0) + +def AddRasChallengeVersion(builder: octoflatbuffers.Builder, rasChallengeVersion: int): + HandshakeSynAddRasChallengeVersion(builder, rasChallengeVersion) + +def HandshakeSynAddWebcamFlipH(builder: octoflatbuffers.Builder, webcamFlipH: bool): + builder.PrependBoolSlot(8, webcamFlipH, 0) + +def AddWebcamFlipH(builder: octoflatbuffers.Builder, webcamFlipH: bool): + HandshakeSynAddWebcamFlipH(builder, webcamFlipH) + +def HandshakeSynAddWebcamFlipV(builder: octoflatbuffers.Builder, webcamFlipV: bool): + builder.PrependBoolSlot(9, webcamFlipV, 0) + +def AddWebcamFlipV(builder: octoflatbuffers.Builder, webcamFlipV: bool): + HandshakeSynAddWebcamFlipV(builder, webcamFlipV) + +def HandshakeSynAddWebcamFlipRotate90(builder: octoflatbuffers.Builder, webcamFlipRotate90: bool): + builder.PrependBoolSlot(10, webcamFlipRotate90, 0) + +def AddWebcamFlipRotate90(builder: octoflatbuffers.Builder, webcamFlipRotate90: bool): + HandshakeSynAddWebcamFlipRotate90(builder, webcamFlipRotate90) + +def HandshakeSynAddPrivateKey(builder: octoflatbuffers.Builder, privateKey: int): + builder.PrependUOffsetTRelativeSlot(11, octoflatbuffers.number_types.UOffsetTFlags.py_type(privateKey), 0) + +def AddPrivateKey(builder: octoflatbuffers.Builder, privateKey: int): + HandshakeSynAddPrivateKey(builder, privateKey) + +def HandshakeSynAddSummonMethod(builder: octoflatbuffers.Builder, summonMethod: int): + builder.PrependInt8Slot(12, summonMethod, 1) + +def AddSummonMethod(builder: octoflatbuffers.Builder, summonMethod: int): + HandshakeSynAddSummonMethod(builder, summonMethod) + +def HandshakeSynAddServerHost(builder: octoflatbuffers.Builder, serverHost: int): + builder.PrependInt8Slot(13, serverHost, 0) + +def AddServerHost(builder: octoflatbuffers.Builder, serverHost: int): + HandshakeSynAddServerHost(builder, serverHost) + +def HandshakeSynAddIsCompanion(builder: octoflatbuffers.Builder, isCompanion: bool): + builder.PrependBoolSlot(14, isCompanion, 0) + +def AddIsCompanion(builder: octoflatbuffers.Builder, isCompanion: bool): + HandshakeSynAddIsCompanion(builder, isCompanion) + +def HandshakeSynAddOsType(builder: octoflatbuffers.Builder, osType: int): + builder.PrependInt8Slot(15, osType, 0) + +def AddOsType(builder: octoflatbuffers.Builder, osType: int): + HandshakeSynAddOsType(builder, osType) + +def HandshakeSynEnd(builder: octoflatbuffers.Builder) -> int: + return builder.EndObject() + +def End(builder: octoflatbuffers.Builder) -> int: + return HandshakeSynEnd(builder) diff --git a/octoeverywhere/Proto/HttpHeader.py b/octoeverywhere/Proto/HttpHeader.py index 4d96400..3bb7b59 100644 --- a/octoeverywhere/Proto/HttpHeader.py +++ b/octoeverywhere/Proto/HttpHeader.py @@ -3,11 +3,13 @@ # namespace: Proto import octoflatbuffers +from typing import Any +from typing import Optional class HttpHeader(object): __slots__ = ['_tab'] @classmethod - def GetRootAs(cls, buf, offset=0): + def GetRootAs(cls, buf, offset: int = 0): n = octoflatbuffers.encode.Get(octoflatbuffers.packer.uoffset, buf, offset) x = HttpHeader() x.Init(buf, n + offset) @@ -18,36 +20,43 @@ def GetRootAsHttpHeader(cls, buf, offset=0): """This method is deprecated. Please switch to GetRootAs.""" return cls.GetRootAs(buf, offset) # HttpHeader - def Init(self, buf, pos): + def Init(self, buf: bytes, pos: int): self._tab = octoflatbuffers.table.Table(buf, pos) # HttpHeader - def Key(self): + def Key(self) -> Optional[str]: o = octoflatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) if o != 0: return self._tab.String(o + self._tab.Pos) return None # HttpHeader - def Value(self): + def Value(self) -> Optional[str]: o = octoflatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) if o != 0: return self._tab.String(o + self._tab.Pos) return None -def Start(builder): builder.StartObject(2) -def HttpHeaderStart(builder): - """This method is deprecated. Please switch to Start.""" - return Start(builder) -def AddKey(builder, key): builder.PrependUOffsetTRelativeSlot(0, octoflatbuffers.number_types.UOffsetTFlags.py_type(key), 0) -def HttpHeaderAddKey(builder, key): - """This method is deprecated. Please switch to AddKey.""" - return AddKey(builder, key) -def AddValue(builder, value): builder.PrependUOffsetTRelativeSlot(1, octoflatbuffers.number_types.UOffsetTFlags.py_type(value), 0) -def HttpHeaderAddValue(builder, value): - """This method is deprecated. Please switch to AddValue.""" - return AddValue(builder, value) -def End(builder): return builder.EndObject() -def HttpHeaderEnd(builder): - """This method is deprecated. Please switch to End.""" - return End(builder) \ No newline at end of file +def HttpHeaderStart(builder: octoflatbuffers.Builder): + builder.StartObject(2) + +def Start(builder: octoflatbuffers.Builder): + HttpHeaderStart(builder) + +def HttpHeaderAddKey(builder: octoflatbuffers.Builder, key: int): + builder.PrependUOffsetTRelativeSlot(0, octoflatbuffers.number_types.UOffsetTFlags.py_type(key), 0) + +def AddKey(builder: octoflatbuffers.Builder, key: int): + HttpHeaderAddKey(builder, key) + +def HttpHeaderAddValue(builder: octoflatbuffers.Builder, value: int): + builder.PrependUOffsetTRelativeSlot(1, octoflatbuffers.number_types.UOffsetTFlags.py_type(value), 0) + +def AddValue(builder: octoflatbuffers.Builder, value: int): + HttpHeaderAddValue(builder, value) + +def HttpHeaderEnd(builder: octoflatbuffers.Builder) -> int: + return builder.EndObject() + +def End(builder: octoflatbuffers.Builder) -> int: + return HttpHeaderEnd(builder) diff --git a/octoeverywhere/Proto/HttpInitialContext.py b/octoeverywhere/Proto/HttpInitialContext.py index 67d1a5f..2d4aa3c 100644 --- a/octoeverywhere/Proto/HttpInitialContext.py +++ b/octoeverywhere/Proto/HttpInitialContext.py @@ -3,11 +3,14 @@ # namespace: Proto import octoflatbuffers +from typing import Any +from octoeverywhere.Proto.HttpHeader import HttpHeader +from typing import Optional class HttpInitialContext(object): __slots__ = ['_tab'] @classmethod - def GetRootAs(cls, buf, offset=0): + def GetRootAs(cls, buf, offset: int = 0): n = octoflatbuffers.encode.Get(octoflatbuffers.packer.uoffset, buf, offset) x = HttpInitialContext() x.Init(buf, n + offset) @@ -18,11 +21,11 @@ def GetRootAsHttpInitialContext(cls, buf, offset=0): """This method is deprecated. Please switch to GetRootAs.""" return cls.GetRootAs(buf, offset) # HttpInitialContext - def Init(self, buf, pos): + def Init(self, buf: bytes, pos: int): self._tab = octoflatbuffers.table.Table(buf, pos) # HttpInitialContext - def Path(self): + def Path(self) -> Optional[str]: o = octoflatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) if o != 0: return self._tab.String(o + self._tab.Pos) @@ -36,41 +39,40 @@ def PathType(self): return 1 # HttpInitialContext - def Method(self): + def Method(self) -> Optional[str]: o = octoflatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8)) if o != 0: return self._tab.String(o + self._tab.Pos) return None # HttpInitialContext - def OctoHost(self): + def OctoHost(self) -> Optional[str]: o = octoflatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10)) if o != 0: return self._tab.String(o + self._tab.Pos) return None # HttpInitialContext - def Headers(self, j): + def Headers(self, j: int) -> Optional[HttpHeader]: o = octoflatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12)) if o != 0: x = self._tab.Vector(o) x += octoflatbuffers.number_types.UOffsetTFlags.py_type(j) * 4 x = self._tab.Indirect(x) - from octoeverywhere.Proto.HttpHeader import HttpHeader obj = HttpHeader() obj.Init(self._tab.Bytes, x) return obj return None # HttpInitialContext - def HeadersLength(self): + def HeadersLength(self) -> int: o = octoflatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12)) if o != 0: return self._tab.VectorLen(o) return 0 # HttpInitialContext - def HeadersIsNone(self): + def HeadersIsNone(self) -> bool: o = octoflatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12)) return o == 0 @@ -81,39 +83,56 @@ def UseOctoeverywhereAuth(self): return self._tab.Get(octoflatbuffers.number_types.Int8Flags, o + self._tab.Pos) return 0 -def Start(builder): builder.StartObject(6) -def HttpInitialContextStart(builder): - """This method is deprecated. Please switch to Start.""" - return Start(builder) -def AddPath(builder, path): builder.PrependUOffsetTRelativeSlot(0, octoflatbuffers.number_types.UOffsetTFlags.py_type(path), 0) -def HttpInitialContextAddPath(builder, path): - """This method is deprecated. Please switch to AddPath.""" - return AddPath(builder, path) -def AddPathType(builder, pathType): builder.PrependInt8Slot(1, pathType, 1) -def HttpInitialContextAddPathType(builder, pathType): - """This method is deprecated. Please switch to AddPathType.""" - return AddPathType(builder, pathType) -def AddMethod(builder, method): builder.PrependUOffsetTRelativeSlot(2, octoflatbuffers.number_types.UOffsetTFlags.py_type(method), 0) -def HttpInitialContextAddMethod(builder, method): - """This method is deprecated. Please switch to AddMethod.""" - return AddMethod(builder, method) -def AddOctoHost(builder, octoHost): builder.PrependUOffsetTRelativeSlot(3, octoflatbuffers.number_types.UOffsetTFlags.py_type(octoHost), 0) -def HttpInitialContextAddOctoHost(builder, octoHost): - """This method is deprecated. Please switch to AddOctoHost.""" - return AddOctoHost(builder, octoHost) -def AddHeaders(builder, headers): builder.PrependUOffsetTRelativeSlot(4, octoflatbuffers.number_types.UOffsetTFlags.py_type(headers), 0) -def HttpInitialContextAddHeaders(builder, headers): - """This method is deprecated. Please switch to AddHeaders.""" - return AddHeaders(builder, headers) -def StartHeadersVector(builder, numElems): return builder.StartVector(4, numElems, 4) -def HttpInitialContextStartHeadersVector(builder, numElems): - """This method is deprecated. Please switch to Start.""" - return StartHeadersVector(builder, numElems) -def AddUseOctoeverywhereAuth(builder, useOctoeverywhereAuth): builder.PrependInt8Slot(5, useOctoeverywhereAuth, 0) -def HttpInitialContextAddUseOctoeverywhereAuth(builder, useOctoeverywhereAuth): - """This method is deprecated. Please switch to AddUseOctoeverywhereAuth.""" - return AddUseOctoeverywhereAuth(builder, useOctoeverywhereAuth) -def End(builder): return builder.EndObject() -def HttpInitialContextEnd(builder): - """This method is deprecated. Please switch to End.""" - return End(builder) \ No newline at end of file +def HttpInitialContextStart(builder: octoflatbuffers.Builder): + builder.StartObject(6) + +def Start(builder: octoflatbuffers.Builder): + HttpInitialContextStart(builder) + +def HttpInitialContextAddPath(builder: octoflatbuffers.Builder, path: int): + builder.PrependUOffsetTRelativeSlot(0, octoflatbuffers.number_types.UOffsetTFlags.py_type(path), 0) + +def AddPath(builder: octoflatbuffers.Builder, path: int): + HttpInitialContextAddPath(builder, path) + +def HttpInitialContextAddPathType(builder: octoflatbuffers.Builder, pathType: int): + builder.PrependInt8Slot(1, pathType, 1) + +def AddPathType(builder: octoflatbuffers.Builder, pathType: int): + HttpInitialContextAddPathType(builder, pathType) + +def HttpInitialContextAddMethod(builder: octoflatbuffers.Builder, method: int): + builder.PrependUOffsetTRelativeSlot(2, octoflatbuffers.number_types.UOffsetTFlags.py_type(method), 0) + +def AddMethod(builder: octoflatbuffers.Builder, method: int): + HttpInitialContextAddMethod(builder, method) + +def HttpInitialContextAddOctoHost(builder: octoflatbuffers.Builder, octoHost: int): + builder.PrependUOffsetTRelativeSlot(3, octoflatbuffers.number_types.UOffsetTFlags.py_type(octoHost), 0) + +def AddOctoHost(builder: octoflatbuffers.Builder, octoHost: int): + HttpInitialContextAddOctoHost(builder, octoHost) + +def HttpInitialContextAddHeaders(builder: octoflatbuffers.Builder, headers: int): + builder.PrependUOffsetTRelativeSlot(4, octoflatbuffers.number_types.UOffsetTFlags.py_type(headers), 0) + +def AddHeaders(builder: octoflatbuffers.Builder, headers: int): + HttpInitialContextAddHeaders(builder, headers) + +def HttpInitialContextStartHeadersVector(builder, numElems: int) -> int: + return builder.StartVector(4, numElems, 4) + +def StartHeadersVector(builder, numElems: int) -> int: + return HttpInitialContextStartHeadersVector(builder, numElems) + +def HttpInitialContextAddUseOctoeverywhereAuth(builder: octoflatbuffers.Builder, useOctoeverywhereAuth: int): + builder.PrependInt8Slot(5, useOctoeverywhereAuth, 0) + +def AddUseOctoeverywhereAuth(builder: octoflatbuffers.Builder, useOctoeverywhereAuth: int): + HttpInitialContextAddUseOctoeverywhereAuth(builder, useOctoeverywhereAuth) + +def HttpInitialContextEnd(builder: octoflatbuffers.Builder) -> int: + return builder.EndObject() + +def End(builder: octoflatbuffers.Builder) -> int: + return HttpInitialContextEnd(builder) diff --git a/octoeverywhere/Proto/MessageContext.py b/octoeverywhere/Proto/MessageContext.py index a00df7d..2a926f9 100644 --- a/octoeverywhere/Proto/MessageContext.py +++ b/octoeverywhere/Proto/MessageContext.py @@ -9,4 +9,3 @@ class MessageContext(object): WebStreamMsg = 3 OctoNotification = 4 OctoSummon = 5 - diff --git a/octoeverywhere/Proto/MessagePriority.py b/octoeverywhere/Proto/MessagePriority.py index baf09a1..5078492 100644 --- a/octoeverywhere/Proto/MessagePriority.py +++ b/octoeverywhere/Proto/MessagePriority.py @@ -8,4 +8,3 @@ class MessagePriority(object): Normal = 10 Low = 15 Background = 20 - diff --git a/octoeverywhere/Proto/OctoNotification.py b/octoeverywhere/Proto/OctoNotification.py index 6575746..3b0c5dc 100644 --- a/octoeverywhere/Proto/OctoNotification.py +++ b/octoeverywhere/Proto/OctoNotification.py @@ -3,11 +3,13 @@ # namespace: Proto import octoflatbuffers +from typing import Any +from typing import Optional class OctoNotification(object): __slots__ = ['_tab'] @classmethod - def GetRootAs(cls, buf, offset=0): + def GetRootAs(cls, buf, offset: int = 0): n = octoflatbuffers.encode.Get(octoflatbuffers.packer.uoffset, buf, offset) x = OctoNotification() x.Init(buf, n + offset) @@ -18,18 +20,18 @@ def GetRootAsOctoNotification(cls, buf, offset=0): """This method is deprecated. Please switch to GetRootAs.""" return cls.GetRootAs(buf, offset) # OctoNotification - def Init(self, buf, pos): + def Init(self, buf: bytes, pos: int): self._tab = octoflatbuffers.table.Table(buf, pos) # OctoNotification - def Title(self): + def Title(self) -> Optional[str]: o = octoflatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) if o != 0: return self._tab.String(o + self._tab.Pos) return None # OctoNotification - def Text(self): + def Text(self) -> Optional[str]: o = octoflatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) if o != 0: return self._tab.String(o + self._tab.Pos) @@ -43,14 +45,14 @@ def Type(self): return 0 # OctoNotification - def ActionText(self): + def ActionText(self) -> Optional[str]: o = octoflatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12)) if o != 0: return self._tab.String(o + self._tab.Pos) return None # OctoNotification - def ActionLink(self): + def ActionLink(self) -> Optional[str]: o = octoflatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14)) if o != 0: return self._tab.String(o + self._tab.Pos) @@ -70,39 +72,56 @@ def ShowOnlyIfLoadedFromOe(self): return bool(self._tab.Get(octoflatbuffers.number_types.BoolFlags, o + self._tab.Pos)) return True -def Start(builder): builder.StartObject(8) -def OctoNotificationStart(builder): - """This method is deprecated. Please switch to Start.""" - return Start(builder) -def AddTitle(builder, title): builder.PrependUOffsetTRelativeSlot(0, octoflatbuffers.number_types.UOffsetTFlags.py_type(title), 0) -def OctoNotificationAddTitle(builder, title): - """This method is deprecated. Please switch to AddTitle.""" - return AddTitle(builder, title) -def AddText(builder, text): builder.PrependUOffsetTRelativeSlot(1, octoflatbuffers.number_types.UOffsetTFlags.py_type(text), 0) -def OctoNotificationAddText(builder, text): - """This method is deprecated. Please switch to AddText.""" - return AddText(builder, text) -def AddType(builder, type): builder.PrependInt8Slot(2, type, 0) -def OctoNotificationAddType(builder, type): - """This method is deprecated. Please switch to AddType.""" - return AddType(builder, type) -def AddActionText(builder, actionText): builder.PrependUOffsetTRelativeSlot(4, octoflatbuffers.number_types.UOffsetTFlags.py_type(actionText), 0) -def OctoNotificationAddActionText(builder, actionText): - """This method is deprecated. Please switch to AddActionText.""" - return AddActionText(builder, actionText) -def AddActionLink(builder, actionLink): builder.PrependUOffsetTRelativeSlot(5, octoflatbuffers.number_types.UOffsetTFlags.py_type(actionLink), 0) -def OctoNotificationAddActionLink(builder, actionLink): - """This method is deprecated. Please switch to AddActionLink.""" - return AddActionLink(builder, actionLink) -def AddShowForSec(builder, showForSec): builder.PrependUint32Slot(6, showForSec, 5) -def OctoNotificationAddShowForSec(builder, showForSec): - """This method is deprecated. Please switch to AddShowForSec.""" - return AddShowForSec(builder, showForSec) -def AddShowOnlyIfLoadedFromOe(builder, showOnlyIfLoadedFromOe): builder.PrependBoolSlot(7, showOnlyIfLoadedFromOe, 1) -def OctoNotificationAddShowOnlyIfLoadedFromOe(builder, showOnlyIfLoadedFromOe): - """This method is deprecated. Please switch to AddShowOnlyIfLoadedFromOe.""" - return AddShowOnlyIfLoadedFromOe(builder, showOnlyIfLoadedFromOe) -def End(builder): return builder.EndObject() -def OctoNotificationEnd(builder): - """This method is deprecated. Please switch to End.""" - return End(builder) \ No newline at end of file +def OctoNotificationStart(builder: octoflatbuffers.Builder): + builder.StartObject(8) + +def Start(builder: octoflatbuffers.Builder): + OctoNotificationStart(builder) + +def OctoNotificationAddTitle(builder: octoflatbuffers.Builder, title: int): + builder.PrependUOffsetTRelativeSlot(0, octoflatbuffers.number_types.UOffsetTFlags.py_type(title), 0) + +def AddTitle(builder: octoflatbuffers.Builder, title: int): + OctoNotificationAddTitle(builder, title) + +def OctoNotificationAddText(builder: octoflatbuffers.Builder, text: int): + builder.PrependUOffsetTRelativeSlot(1, octoflatbuffers.number_types.UOffsetTFlags.py_type(text), 0) + +def AddText(builder: octoflatbuffers.Builder, text: int): + OctoNotificationAddText(builder, text) + +def OctoNotificationAddType(builder: octoflatbuffers.Builder, type: int): + builder.PrependInt8Slot(2, type, 0) + +def AddType(builder: octoflatbuffers.Builder, type: int): + OctoNotificationAddType(builder, type) + +def OctoNotificationAddActionText(builder: octoflatbuffers.Builder, actionText: int): + builder.PrependUOffsetTRelativeSlot(4, octoflatbuffers.number_types.UOffsetTFlags.py_type(actionText), 0) + +def AddActionText(builder: octoflatbuffers.Builder, actionText: int): + OctoNotificationAddActionText(builder, actionText) + +def OctoNotificationAddActionLink(builder: octoflatbuffers.Builder, actionLink: int): + builder.PrependUOffsetTRelativeSlot(5, octoflatbuffers.number_types.UOffsetTFlags.py_type(actionLink), 0) + +def AddActionLink(builder: octoflatbuffers.Builder, actionLink: int): + OctoNotificationAddActionLink(builder, actionLink) + +def OctoNotificationAddShowForSec(builder: octoflatbuffers.Builder, showForSec: int): + builder.PrependUint32Slot(6, showForSec, 5) + +def AddShowForSec(builder: octoflatbuffers.Builder, showForSec: int): + OctoNotificationAddShowForSec(builder, showForSec) + +def OctoNotificationAddShowOnlyIfLoadedFromOe(builder: octoflatbuffers.Builder, showOnlyIfLoadedFromOe: bool): + builder.PrependBoolSlot(7, showOnlyIfLoadedFromOe, 1) + +def AddShowOnlyIfLoadedFromOe(builder: octoflatbuffers.Builder, showOnlyIfLoadedFromOe: bool): + OctoNotificationAddShowOnlyIfLoadedFromOe(builder, showOnlyIfLoadedFromOe) + +def OctoNotificationEnd(builder: octoflatbuffers.Builder) -> int: + return builder.EndObject() + +def End(builder: octoflatbuffers.Builder) -> int: + return OctoNotificationEnd(builder) diff --git a/octoeverywhere/Proto/OctoNotificationTypes.py b/octoeverywhere/Proto/OctoNotificationTypes.py index f8fc3ba..1a581b3 100644 --- a/octoeverywhere/Proto/OctoNotificationTypes.py +++ b/octoeverywhere/Proto/OctoNotificationTypes.py @@ -7,4 +7,3 @@ class OctoNotificationTypes(object): Info = 1 Success = 2 Error = 3 - diff --git a/octoeverywhere/Proto/OctoStreamMessage.py b/octoeverywhere/Proto/OctoStreamMessage.py index f0f6805..84ac1a0 100644 --- a/octoeverywhere/Proto/OctoStreamMessage.py +++ b/octoeverywhere/Proto/OctoStreamMessage.py @@ -3,11 +3,14 @@ # namespace: Proto import octoflatbuffers +from typing import Any +from octoflatbuffers.table import Table +from typing import Optional class OctoStreamMessage(object): __slots__ = ['_tab'] @classmethod - def GetRootAs(cls, buf, offset=0): + def GetRootAs(cls, buf, offset: int = 0): n = octoflatbuffers.encode.Get(octoflatbuffers.packer.uoffset, buf, offset) x = OctoStreamMessage() x.Init(buf, n + offset) @@ -18,7 +21,7 @@ def GetRootAsOctoStreamMessage(cls, buf, offset=0): """This method is deprecated. Please switch to GetRootAs.""" return cls.GetRootAs(buf, offset) # OctoStreamMessage - def Init(self, buf, pos): + def Init(self, buf: bytes, pos: int): self._tab = octoflatbuffers.table.Table(buf, pos) # OctoStreamMessage @@ -29,28 +32,34 @@ def ContextType(self): return 0 # OctoStreamMessage - def Context(self): + def Context(self) -> Optional[octoflatbuffers.table.Table]: o = octoflatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) if o != 0: - from octoflatbuffers.table import Table obj = Table(bytearray(), 0) self._tab.Union(obj, o) return obj return None -def Start(builder): builder.StartObject(2) -def OctoStreamMessageStart(builder): - """This method is deprecated. Please switch to Start.""" - return Start(builder) -def AddContextType(builder, contextType): builder.PrependUint8Slot(0, contextType, 0) -def OctoStreamMessageAddContextType(builder, contextType): - """This method is deprecated. Please switch to AddContextType.""" - return AddContextType(builder, contextType) -def AddContext(builder, context): builder.PrependUOffsetTRelativeSlot(1, octoflatbuffers.number_types.UOffsetTFlags.py_type(context), 0) -def OctoStreamMessageAddContext(builder, context): - """This method is deprecated. Please switch to AddContext.""" - return AddContext(builder, context) -def End(builder): return builder.EndObject() -def OctoStreamMessageEnd(builder): - """This method is deprecated. Please switch to End.""" - return End(builder) \ No newline at end of file +def OctoStreamMessageStart(builder: octoflatbuffers.Builder): + builder.StartObject(2) + +def Start(builder: octoflatbuffers.Builder): + OctoStreamMessageStart(builder) + +def OctoStreamMessageAddContextType(builder: octoflatbuffers.Builder, contextType: int): + builder.PrependUint8Slot(0, contextType, 0) + +def AddContextType(builder: octoflatbuffers.Builder, contextType: int): + OctoStreamMessageAddContextType(builder, contextType) + +def OctoStreamMessageAddContext(builder: octoflatbuffers.Builder, context: int): + builder.PrependUOffsetTRelativeSlot(1, octoflatbuffers.number_types.UOffsetTFlags.py_type(context), 0) + +def AddContext(builder: octoflatbuffers.Builder, context: int): + OctoStreamMessageAddContext(builder, context) + +def OctoStreamMessageEnd(builder: octoflatbuffers.Builder) -> int: + return builder.EndObject() + +def End(builder: octoflatbuffers.Builder) -> int: + return OctoStreamMessageEnd(builder) diff --git a/octoeverywhere/Proto/OctoSummon.py b/octoeverywhere/Proto/OctoSummon.py index 3820fef..9fdac55 100644 --- a/octoeverywhere/Proto/OctoSummon.py +++ b/octoeverywhere/Proto/OctoSummon.py @@ -3,11 +3,13 @@ # namespace: Proto import octoflatbuffers +from typing import Any +from typing import Optional class OctoSummon(object): __slots__ = ['_tab'] @classmethod - def GetRootAs(cls, buf, offset=0): + def GetRootAs(cls, buf, offset: int = 0): n = octoflatbuffers.encode.Get(octoflatbuffers.packer.uoffset, buf, offset) x = OctoSummon() x.Init(buf, n + offset) @@ -18,11 +20,11 @@ def GetRootAsOctoSummon(cls, buf, offset=0): """This method is deprecated. Please switch to GetRootAs.""" return cls.GetRootAs(buf, offset) # OctoSummon - def Init(self, buf, pos): + def Init(self, buf: bytes, pos: int): self._tab = octoflatbuffers.table.Table(buf, pos) # OctoSummon - def ServerConnectUrl(self): + def ServerConnectUrl(self) -> Optional[str]: o = octoflatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) if o != 0: return self._tab.String(o + self._tab.Pos) @@ -35,19 +37,26 @@ def SummonMethod(self): return self._tab.Get(octoflatbuffers.number_types.Int8Flags, o + self._tab.Pos) return 1 -def Start(builder): builder.StartObject(2) -def OctoSummonStart(builder): - """This method is deprecated. Please switch to Start.""" - return Start(builder) -def AddServerConnectUrl(builder, serverConnectUrl): builder.PrependUOffsetTRelativeSlot(0, octoflatbuffers.number_types.UOffsetTFlags.py_type(serverConnectUrl), 0) -def OctoSummonAddServerConnectUrl(builder, serverConnectUrl): - """This method is deprecated. Please switch to AddServerConnectUrl.""" - return AddServerConnectUrl(builder, serverConnectUrl) -def AddSummonMethod(builder, summonMethod): builder.PrependInt8Slot(1, summonMethod, 1) -def OctoSummonAddSummonMethod(builder, summonMethod): - """This method is deprecated. Please switch to AddSummonMethod.""" - return AddSummonMethod(builder, summonMethod) -def End(builder): return builder.EndObject() -def OctoSummonEnd(builder): - """This method is deprecated. Please switch to End.""" - return End(builder) \ No newline at end of file +def OctoSummonStart(builder: octoflatbuffers.Builder): + builder.StartObject(2) + +def Start(builder: octoflatbuffers.Builder): + OctoSummonStart(builder) + +def OctoSummonAddServerConnectUrl(builder: octoflatbuffers.Builder, serverConnectUrl: int): + builder.PrependUOffsetTRelativeSlot(0, octoflatbuffers.number_types.UOffsetTFlags.py_type(serverConnectUrl), 0) + +def AddServerConnectUrl(builder: octoflatbuffers.Builder, serverConnectUrl: int): + OctoSummonAddServerConnectUrl(builder, serverConnectUrl) + +def OctoSummonAddSummonMethod(builder: octoflatbuffers.Builder, summonMethod: int): + builder.PrependInt8Slot(1, summonMethod, 1) + +def AddSummonMethod(builder: octoflatbuffers.Builder, summonMethod: int): + OctoSummonAddSummonMethod(builder, summonMethod) + +def OctoSummonEnd(builder: octoflatbuffers.Builder) -> int: + return builder.EndObject() + +def End(builder: octoflatbuffers.Builder) -> int: + return OctoSummonEnd(builder) diff --git a/octoeverywhere/Proto/OeAuthAllowed.py b/octoeverywhere/Proto/OeAuthAllowed.py index 2dc745d..c3abd59 100644 --- a/octoeverywhere/Proto/OeAuthAllowed.py +++ b/octoeverywhere/Proto/OeAuthAllowed.py @@ -5,4 +5,3 @@ class OeAuthAllowed(object): Deny = 0 Allow = 1 - diff --git a/octoeverywhere/Proto/OsType.py b/octoeverywhere/Proto/OsType.py index 8e229f1..e4a7d58 100644 --- a/octoeverywhere/Proto/OsType.py +++ b/octoeverywhere/Proto/OsType.py @@ -8,4 +8,3 @@ class OsType(object): Windows = 2 CrealitySonicPad = 3 CrealityK1 = 4 - diff --git a/octoeverywhere/Proto/PathTypes.py b/octoeverywhere/Proto/PathTypes.py index 039cf97..fdba7fb 100644 --- a/octoeverywhere/Proto/PathTypes.py +++ b/octoeverywhere/Proto/PathTypes.py @@ -6,4 +6,3 @@ class PathTypes(object): None_ = 0 Relative = 1 Absolute = 2 - diff --git a/octoeverywhere/Proto/ServerHost.py b/octoeverywhere/Proto/ServerHost.py index 340d5a0..ab5d96d 100644 --- a/octoeverywhere/Proto/ServerHost.py +++ b/octoeverywhere/Proto/ServerHost.py @@ -7,4 +7,3 @@ class ServerHost(object): OctoPrint = 1 Moonraker = 2 Bambu = 3 - diff --git a/octoeverywhere/Proto/SummonMethods.py b/octoeverywhere/Proto/SummonMethods.py index 16204e8..ac5e278 100644 --- a/octoeverywhere/Proto/SummonMethods.py +++ b/octoeverywhere/Proto/SummonMethods.py @@ -6,4 +6,3 @@ class SummonMethods(object): Unknown = 1 FastPath = 2 Broadcast = 3 - diff --git a/octoeverywhere/Proto/WebSocketDataTypes.py b/octoeverywhere/Proto/WebSocketDataTypes.py index e59fefe..146c375 100644 --- a/octoeverywhere/Proto/WebSocketDataTypes.py +++ b/octoeverywhere/Proto/WebSocketDataTypes.py @@ -7,4 +7,3 @@ class WebSocketDataTypes(object): Binary = 2 Close = 8 None_ = 126 - diff --git a/octoeverywhere/Proto/WebStreamMsg.py b/octoeverywhere/Proto/WebStreamMsg.py index 7bcfcd5..516f7ed 100644 --- a/octoeverywhere/Proto/WebStreamMsg.py +++ b/octoeverywhere/Proto/WebStreamMsg.py @@ -3,11 +3,14 @@ # namespace: Proto import octoflatbuffers +from typing import Any +from octoeverywhere.Proto.HttpInitialContext import HttpInitialContext +from typing import Optional class WebStreamMsg(object): __slots__ = ['_tab'] @classmethod - def GetRootAs(cls, buf, offset=0): + def GetRootAs(cls, buf, offset: int = 0): n = octoflatbuffers.encode.Get(octoflatbuffers.packer.uoffset, buf, offset) x = WebStreamMsg() x.Init(buf, n + offset) @@ -18,7 +21,7 @@ def GetRootAsWebStreamMsg(cls, buf, offset=0): """This method is deprecated. Please switch to GetRootAs.""" return cls.GetRootAs(buf, offset) # WebStreamMsg - def Init(self, buf, pos): + def Init(self, buf: bytes, pos: int): self._tab = octoflatbuffers.table.Table(buf, pos) # WebStreamMsg @@ -64,7 +67,7 @@ def FullStreamDataSize(self): return -1 # WebStreamMsg - def Data(self, j): + def Data(self, j: int): o = octoflatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(16)) if o != 0: a = self._tab.Vector(o) @@ -86,14 +89,14 @@ def DataAsByteArray(self): return 0 # WebStreamMsg - def DataLength(self): + def DataLength(self) -> int: o = octoflatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(16)) if o != 0: return self._tab.VectorLen(o) return 0 # WebStreamMsg - def DataIsNone(self): + def DataIsNone(self) -> bool: o = octoflatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(16)) return o == 0 @@ -112,11 +115,10 @@ def OriginalDataSize(self): return 0 # WebStreamMsg - def HttpInitialContext(self): + def HttpInitialContext(self) -> Optional[HttpInitialContext]: o = octoflatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(22)) if o != 0: x = self._tab.Indirect(o + self._tab.Pos) - from octoeverywhere.Proto.HttpInitialContext import HttpInitialContext obj = HttpInitialContext() obj.Init(self._tab.Bytes, x) return obj @@ -178,87 +180,128 @@ def MultipartReadsPerSecond(self): return self._tab.Get(octoflatbuffers.number_types.Uint8Flags, o + self._tab.Pos) return 0 -def Start(builder): builder.StartObject(18) -def WebStreamMsgStart(builder): - """This method is deprecated. Please switch to Start.""" - return Start(builder) -def AddStreamId(builder, streamId): builder.PrependUint32Slot(0, streamId, 0) -def WebStreamMsgAddStreamId(builder, streamId): - """This method is deprecated. Please switch to AddStreamId.""" - return AddStreamId(builder, streamId) -def AddIsOpenMsg(builder, isOpenMsg): builder.PrependBoolSlot(1, isOpenMsg, 0) -def WebStreamMsgAddIsOpenMsg(builder, isOpenMsg): - """This method is deprecated. Please switch to AddIsOpenMsg.""" - return AddIsOpenMsg(builder, isOpenMsg) -def AddIsCloseMsg(builder, isCloseMsg): builder.PrependBoolSlot(2, isCloseMsg, 0) -def WebStreamMsgAddIsCloseMsg(builder, isCloseMsg): - """This method is deprecated. Please switch to AddIsCloseMsg.""" - return AddIsCloseMsg(builder, isCloseMsg) -def AddIsDataTransmissionDone(builder, isDataTransmissionDone): builder.PrependBoolSlot(3, isDataTransmissionDone, 0) -def WebStreamMsgAddIsDataTransmissionDone(builder, isDataTransmissionDone): - """This method is deprecated. Please switch to AddIsDataTransmissionDone.""" - return AddIsDataTransmissionDone(builder, isDataTransmissionDone) -def AddIsControlFlagsOnly(builder, isControlFlagsOnly): builder.PrependBoolSlot(4, isControlFlagsOnly, 1) -def WebStreamMsgAddIsControlFlagsOnly(builder, isControlFlagsOnly): - """This method is deprecated. Please switch to AddIsControlFlagsOnly.""" - return AddIsControlFlagsOnly(builder, isControlFlagsOnly) -def AddFullStreamDataSize(builder, fullStreamDataSize): builder.PrependInt64Slot(5, fullStreamDataSize, -1) -def WebStreamMsgAddFullStreamDataSize(builder, fullStreamDataSize): - """This method is deprecated. Please switch to AddFullStreamDataSize.""" - return AddFullStreamDataSize(builder, fullStreamDataSize) -def AddData(builder, data): builder.PrependUOffsetTRelativeSlot(6, octoflatbuffers.number_types.UOffsetTFlags.py_type(data), 0) -def WebStreamMsgAddData(builder, data): - """This method is deprecated. Please switch to AddData.""" - return AddData(builder, data) -def StartDataVector(builder, numElems): return builder.StartVector(1, numElems, 1) -def WebStreamMsgStartDataVector(builder, numElems): - """This method is deprecated. Please switch to Start.""" - return StartDataVector(builder, numElems) -def AddDataCompression(builder, dataCompression): builder.PrependInt8Slot(7, dataCompression, 0) -def WebStreamMsgAddDataCompression(builder, dataCompression): - """This method is deprecated. Please switch to AddDataCompression.""" - return AddDataCompression(builder, dataCompression) -def AddOriginalDataSize(builder, originalDataSize): builder.PrependUint64Slot(8, originalDataSize, 0) -def WebStreamMsgAddOriginalDataSize(builder, originalDataSize): - """This method is deprecated. Please switch to AddOriginalDataSize.""" - return AddOriginalDataSize(builder, originalDataSize) -def AddHttpInitialContext(builder, httpInitialContext): builder.PrependUOffsetTRelativeSlot(9, octoflatbuffers.number_types.UOffsetTFlags.py_type(httpInitialContext), 0) -def WebStreamMsgAddHttpInitialContext(builder, httpInitialContext): - """This method is deprecated. Please switch to AddHttpInitialContext.""" - return AddHttpInitialContext(builder, httpInitialContext) -def AddIsWebsocketStream(builder, isWebsocketStream): builder.PrependBoolSlot(10, isWebsocketStream, 0) -def WebStreamMsgAddIsWebsocketStream(builder, isWebsocketStream): - """This method is deprecated. Please switch to AddIsWebsocketStream.""" - return AddIsWebsocketStream(builder, isWebsocketStream) -def AddStatusCode(builder, statusCode): builder.PrependUint16Slot(11, statusCode, 0) -def WebStreamMsgAddStatusCode(builder, statusCode): - """This method is deprecated. Please switch to AddStatusCode.""" - return AddStatusCode(builder, statusCode) -def AddWebsocketDataType(builder, websocketDataType): builder.PrependInt8Slot(12, websocketDataType, 126) -def WebStreamMsgAddWebsocketDataType(builder, websocketDataType): - """This method is deprecated. Please switch to AddWebsocketDataType.""" - return AddWebsocketDataType(builder, websocketDataType) -def AddMsgPriority(builder, msgPriority): builder.PrependInt8Slot(13, msgPriority, 10) -def WebStreamMsgAddMsgPriority(builder, msgPriority): - """This method is deprecated. Please switch to AddMsgPriority.""" - return AddMsgPriority(builder, msgPriority) -def AddCloseDueToRequestConnectionFailure(builder, closeDueToRequestConnectionFailure): builder.PrependBoolSlot(14, closeDueToRequestConnectionFailure, 0) -def WebStreamMsgAddCloseDueToRequestConnectionFailure(builder, closeDueToRequestConnectionFailure): - """This method is deprecated. Please switch to AddCloseDueToRequestConnectionFailure.""" - return AddCloseDueToRequestConnectionFailure(builder, closeDueToRequestConnectionFailure) -def AddBodyReadTimeHighWaterMarkMs(builder, bodyReadTimeHighWaterMarkMs): builder.PrependUint16Slot(15, bodyReadTimeHighWaterMarkMs, 0) -def WebStreamMsgAddBodyReadTimeHighWaterMarkMs(builder, bodyReadTimeHighWaterMarkMs): - """This method is deprecated. Please switch to AddBodyReadTimeHighWaterMarkMs.""" - return AddBodyReadTimeHighWaterMarkMs(builder, bodyReadTimeHighWaterMarkMs) -def AddSocketSendTimeHighWaterMarkMs(builder, socketSendTimeHighWaterMarkMs): builder.PrependUint16Slot(16, socketSendTimeHighWaterMarkMs, 0) -def WebStreamMsgAddSocketSendTimeHighWaterMarkMs(builder, socketSendTimeHighWaterMarkMs): - """This method is deprecated. Please switch to AddSocketSendTimeHighWaterMarkMs.""" - return AddSocketSendTimeHighWaterMarkMs(builder, socketSendTimeHighWaterMarkMs) -def AddMultipartReadsPerSecond(builder, multipartReadsPerSecond): builder.PrependUint8Slot(17, multipartReadsPerSecond, 0) -def WebStreamMsgAddMultipartReadsPerSecond(builder, multipartReadsPerSecond): - """This method is deprecated. Please switch to AddMultipartReadsPerSecond.""" - return AddMultipartReadsPerSecond(builder, multipartReadsPerSecond) -def End(builder): return builder.EndObject() -def WebStreamMsgEnd(builder): - """This method is deprecated. Please switch to End.""" - return End(builder) \ No newline at end of file +def WebStreamMsgStart(builder: octoflatbuffers.Builder): + builder.StartObject(18) + +def Start(builder: octoflatbuffers.Builder): + WebStreamMsgStart(builder) + +def WebStreamMsgAddStreamId(builder: octoflatbuffers.Builder, streamId: int): + builder.PrependUint32Slot(0, streamId, 0) + +def AddStreamId(builder: octoflatbuffers.Builder, streamId: int): + WebStreamMsgAddStreamId(builder, streamId) + +def WebStreamMsgAddIsOpenMsg(builder: octoflatbuffers.Builder, isOpenMsg: bool): + builder.PrependBoolSlot(1, isOpenMsg, 0) + +def AddIsOpenMsg(builder: octoflatbuffers.Builder, isOpenMsg: bool): + WebStreamMsgAddIsOpenMsg(builder, isOpenMsg) + +def WebStreamMsgAddIsCloseMsg(builder: octoflatbuffers.Builder, isCloseMsg: bool): + builder.PrependBoolSlot(2, isCloseMsg, 0) + +def AddIsCloseMsg(builder: octoflatbuffers.Builder, isCloseMsg: bool): + WebStreamMsgAddIsCloseMsg(builder, isCloseMsg) + +def WebStreamMsgAddIsDataTransmissionDone(builder: octoflatbuffers.Builder, isDataTransmissionDone: bool): + builder.PrependBoolSlot(3, isDataTransmissionDone, 0) + +def AddIsDataTransmissionDone(builder: octoflatbuffers.Builder, isDataTransmissionDone: bool): + WebStreamMsgAddIsDataTransmissionDone(builder, isDataTransmissionDone) + +def WebStreamMsgAddIsControlFlagsOnly(builder: octoflatbuffers.Builder, isControlFlagsOnly: bool): + builder.PrependBoolSlot(4, isControlFlagsOnly, 1) + +def AddIsControlFlagsOnly(builder: octoflatbuffers.Builder, isControlFlagsOnly: bool): + WebStreamMsgAddIsControlFlagsOnly(builder, isControlFlagsOnly) + +def WebStreamMsgAddFullStreamDataSize(builder: octoflatbuffers.Builder, fullStreamDataSize: int): + builder.PrependInt64Slot(5, fullStreamDataSize, -1) + +def AddFullStreamDataSize(builder: octoflatbuffers.Builder, fullStreamDataSize: int): + WebStreamMsgAddFullStreamDataSize(builder, fullStreamDataSize) + +def WebStreamMsgAddData(builder: octoflatbuffers.Builder, data: int): + builder.PrependUOffsetTRelativeSlot(6, octoflatbuffers.number_types.UOffsetTFlags.py_type(data), 0) + +def AddData(builder: octoflatbuffers.Builder, data: int): + WebStreamMsgAddData(builder, data) + +def WebStreamMsgStartDataVector(builder, numElems: int) -> int: + return builder.StartVector(1, numElems, 1) + +def StartDataVector(builder, numElems: int) -> int: + return WebStreamMsgStartDataVector(builder, numElems) + +def WebStreamMsgAddDataCompression(builder: octoflatbuffers.Builder, dataCompression: int): + builder.PrependInt8Slot(7, dataCompression, 0) + +def AddDataCompression(builder: octoflatbuffers.Builder, dataCompression: int): + WebStreamMsgAddDataCompression(builder, dataCompression) + +def WebStreamMsgAddOriginalDataSize(builder: octoflatbuffers.Builder, originalDataSize: int): + builder.PrependUint64Slot(8, originalDataSize, 0) + +def AddOriginalDataSize(builder: octoflatbuffers.Builder, originalDataSize: int): + WebStreamMsgAddOriginalDataSize(builder, originalDataSize) + +def WebStreamMsgAddHttpInitialContext(builder: octoflatbuffers.Builder, httpInitialContext: int): + builder.PrependUOffsetTRelativeSlot(9, octoflatbuffers.number_types.UOffsetTFlags.py_type(httpInitialContext), 0) + +def AddHttpInitialContext(builder: octoflatbuffers.Builder, httpInitialContext: int): + WebStreamMsgAddHttpInitialContext(builder, httpInitialContext) + +def WebStreamMsgAddIsWebsocketStream(builder: octoflatbuffers.Builder, isWebsocketStream: bool): + builder.PrependBoolSlot(10, isWebsocketStream, 0) + +def AddIsWebsocketStream(builder: octoflatbuffers.Builder, isWebsocketStream: bool): + WebStreamMsgAddIsWebsocketStream(builder, isWebsocketStream) + +def WebStreamMsgAddStatusCode(builder: octoflatbuffers.Builder, statusCode: int): + builder.PrependUint16Slot(11, statusCode, 0) + +def AddStatusCode(builder: octoflatbuffers.Builder, statusCode: int): + WebStreamMsgAddStatusCode(builder, statusCode) + +def WebStreamMsgAddWebsocketDataType(builder: octoflatbuffers.Builder, websocketDataType: int): + builder.PrependInt8Slot(12, websocketDataType, 126) + +def AddWebsocketDataType(builder: octoflatbuffers.Builder, websocketDataType: int): + WebStreamMsgAddWebsocketDataType(builder, websocketDataType) + +def WebStreamMsgAddMsgPriority(builder: octoflatbuffers.Builder, msgPriority: int): + builder.PrependInt8Slot(13, msgPriority, 10) + +def AddMsgPriority(builder: octoflatbuffers.Builder, msgPriority: int): + WebStreamMsgAddMsgPriority(builder, msgPriority) + +def WebStreamMsgAddCloseDueToRequestConnectionFailure(builder: octoflatbuffers.Builder, closeDueToRequestConnectionFailure: bool): + builder.PrependBoolSlot(14, closeDueToRequestConnectionFailure, 0) + +def AddCloseDueToRequestConnectionFailure(builder: octoflatbuffers.Builder, closeDueToRequestConnectionFailure: bool): + WebStreamMsgAddCloseDueToRequestConnectionFailure(builder, closeDueToRequestConnectionFailure) + +def WebStreamMsgAddBodyReadTimeHighWaterMarkMs(builder: octoflatbuffers.Builder, bodyReadTimeHighWaterMarkMs: int): + builder.PrependUint16Slot(15, bodyReadTimeHighWaterMarkMs, 0) + +def AddBodyReadTimeHighWaterMarkMs(builder: octoflatbuffers.Builder, bodyReadTimeHighWaterMarkMs: int): + WebStreamMsgAddBodyReadTimeHighWaterMarkMs(builder, bodyReadTimeHighWaterMarkMs) + +def WebStreamMsgAddSocketSendTimeHighWaterMarkMs(builder: octoflatbuffers.Builder, socketSendTimeHighWaterMarkMs: int): + builder.PrependUint16Slot(16, socketSendTimeHighWaterMarkMs, 0) + +def AddSocketSendTimeHighWaterMarkMs(builder: octoflatbuffers.Builder, socketSendTimeHighWaterMarkMs: int): + WebStreamMsgAddSocketSendTimeHighWaterMarkMs(builder, socketSendTimeHighWaterMarkMs) + +def WebStreamMsgAddMultipartReadsPerSecond(builder: octoflatbuffers.Builder, multipartReadsPerSecond: int): + builder.PrependUint8Slot(17, multipartReadsPerSecond, 0) + +def AddMultipartReadsPerSecond(builder: octoflatbuffers.Builder, multipartReadsPerSecond: int): + WebStreamMsgAddMultipartReadsPerSecond(builder, multipartReadsPerSecond) + +def WebStreamMsgEnd(builder: octoflatbuffers.Builder) -> int: + return builder.EndObject() + +def End(builder: octoflatbuffers.Builder) -> int: + return WebStreamMsgEnd(builder) diff --git a/requirements.txt b/requirements.txt index a2362ce..3ce1612 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ # websocket_client>=1.6.0,<1.7.99 requests>=2.31.0 -octoflatbuffers==2.0.5 +octoflatbuffers==24.3.26 pillow certifi>=2023.11.17 rsa>=4.9 diff --git a/setup.py b/setup.py index debee77..079363f 100644 --- a/setup.py +++ b/setup.py @@ -76,7 +76,7 @@ plugin_requires = [ "websocket_client>=1.6.0,<1.7.99", "requests>=2.31.0", - "octoflatbuffers==2.0.5", + "octoflatbuffers==24.3.26", "pillow", "certifi>=2023.11.17", "rsa>=4.9", From f08dcc762bec5918446a1cf8923eb815af4a5f5a Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Wed, 24 Apr 2024 22:02:24 -0700 Subject: [PATCH 077/328] Version bump. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 079363f..69ed4a9 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this is also parsed by the moonraker module to pull the version, so the string and format must remain the same! -plugin_version = "3.2.7" +plugin_version = "3.2.8" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 1228ed4acf05facfafe2d8022623122caeca1d4f Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Mon, 29 Apr 2024 22:34:33 -0700 Subject: [PATCH 078/328] Adding logic to finish the default webcam and alternative webcam features. --- bambu_octoeverywhere/bambuwebcamhelper.py | 22 ++- octoeverywhere/commandhandler.py | 59 +++++-- octoeverywhere/webcamhelper.py | 186 ++++++++++++++++++---- setup.py | 2 +- 4 files changed, 219 insertions(+), 50 deletions(-) diff --git a/bambu_octoeverywhere/bambuwebcamhelper.py b/bambu_octoeverywhere/bambuwebcamhelper.py index 016a5d8..2eb463c 100644 --- a/bambu_octoeverywhere/bambuwebcamhelper.py +++ b/bambu_octoeverywhere/bambuwebcamhelper.py @@ -30,18 +30,24 @@ def __init__(self, logger:logging.Logger, config:Config) -> None: # The order the webcams are returned is the order the user will see in any selection UIs. # Returns None on failure. def GetWebcamConfig(self): - # Bambu has a special webcam setup where there's only one cam and we need to get in a special way, - # So we return this one default webcam object. - return [WebcamSettingItem("Default", BambuWebcamHelper.c_SpecialMockSnapshotPath, BambuWebcamHelper.c_SpecialMockStreamPath, False, False, 0)] + # Bambu has a special webcam setup where there's only one cam and we need to get in a special way, so we return this one default webcam object. + # We do support plugin local webcam items, which are webcams the user can setup from the website and are external webcams. + # Note! This webcam name is shown on to the user in the UI, so it should be a good name that indicates this is a Bambu built in webcam. + # Also, if the name changes, the default printer index might also change. + return [WebcamSettingItem("Bambu Cam", BambuWebcamHelper.c_SpecialMockSnapshotPath, BambuWebcamHelper.c_SpecialMockStreamPath, False, False, 0)] # !! Optional Interface Function !! # If defined, this function must handle ALL snapshot requests for the platform. # - # On failure, return None + # On failure or if this camera doesn't need the override, return None # On success, this will return a valid OctoHttpRequest that's fully filled out. # The snapshot will always already be fully read, and will be FullBodyBuffer var. - def GetSnapshot_Override(self, cameraIndex:int): + def GetSnapshot_Override(self, webcamSettingsItem:WebcamSettingItem): + # Detect if this request is for our special snapshot logic, if not, return None to use the default webcam handling. + if webcamSettingsItem.SnapshotUrl != BambuWebcamHelper.c_SpecialMockSnapshotPath: + return None + # Try to get a snapshot from our QuickCam system. img = QuickCam.Get().GetCurrentImage() if img is None: @@ -60,7 +66,11 @@ def GetSnapshot_Override(self, cameraIndex:int): # On failure, return None # On success, this will return a valid OctoHttpRequest that's fully filled out. # This must return an OctoHttpRequest object with a custom body read stream. - def GetStream_Override(self, cameraIndex:int): + def GetStream_Override(self, webcamSettingsItem:WebcamSettingItem): + # Detect if this request is for our special snapshot logic, if not, return None to use the default webcam handling. + if webcamSettingsItem.StreamUrl != BambuWebcamHelper.c_SpecialMockStreamPath: + return None + # We must create a new instance of this class per stream to ensure all of the vars stay in it's context # and the streams are cleaned up properly. sm = StreamInstance(self.Logger) diff --git a/octoeverywhere/commandhandler.py b/octoeverywhere/commandhandler.py index 9b11647..993a14c 100644 --- a/octoeverywhere/commandhandler.py +++ b/octoeverywhere/commandhandler.py @@ -3,7 +3,7 @@ from .octostreammsgbuilder import OctoStreamMsgBuilder from .octohttprequest import OctoHttpRequest from .octohttprequest import PathTypes -from .webcamhelper import WebcamHelper +from .webcamhelper import WebcamHelper, WebcamSettingItem from .sentry import Sentry # @@ -177,15 +177,7 @@ def ListWebcams(self, includeUrls = True): # Note this format is also used for GetStatus! webcams = [] for i in webcamSettingsItems: - wc = {} - wc["Name"] = i.Name - wc["FlipH"] = i.FlipH - wc["FlipV"] = i.FlipV - wc["Rotation"] = i.Rotation - if includeUrls: - wc["SnapshotUrl"] = i.SnapshotUrl - wc["StreamUrl"] = i.StreamUrl - webcams.append(wc) + webcams.append(i.Serialize(includeUrls)) # We always use the default index, which is a reflection of the current camera list. # We don't use the name, we only use that internally to keep track of the current index. @@ -215,6 +207,49 @@ def SetDefaultCameraName(self, jsonObjData_CanBeNone): return CommandResponse.Success({}) + # Must return a CommandResponse + def GetPluginLocalWebcamSettingsItems(self, _): + # Get the list, make sure we also include any disabled items + localWebcams = WebcamHelper.Get().GetPluginLocalWebcamList(returnDisabledItems=True) + + # Serialize them + webcamDicts = [] + for i in localWebcams: + webcamDicts.append(i.Serialize()) + + # Build the final response + responseObj = { + "LocalPluginWebcams" : webcamDicts, + } + return CommandResponse.Success(responseObj) + + + # Must return a CommandResponse + def SetPluginLocalWebcamSettingsItems(self, jsonObjData_CanBeNone): + localWebcamSettingItems = [] + try: + # Get the list. + items = jsonObjData_CanBeNone.get("LocalPluginWebcams", None) + if items is None: + raise Exception("No LocalPluginWebcams found") + + # Convert the list to objects + for i in items: + # This will deserialize the dict and also validate. + # If None is returned, the item failed, and we won't try to set anything. + o = WebcamSettingItem.Deserialize(i, self.Logger) + if o is None: + raise Exception("Failed to deserialize item") + localWebcamSettingItems.append(o) + except Exception as e: + Sentry.Exception("Failed to SetPluginLocalWebcamSettingsItems, bad args.", e) + return CommandResponse.Error(400, "Failed to parse args") + + # Set the new list + WebcamHelper.Get().SetPluginLocalWebcamList(localWebcamSettingItems) + return CommandResponse.Success({}) + + # Must return a CommandResponse def Pause(self, jsonObjData_CanBeNone): @@ -360,6 +395,10 @@ def ProcessCommand(self, commandPath, jsonObj_CanBeNone): return self.ListWebcams() elif commandPathLower.startswith("set-default-webcam"): return self.SetDefaultCameraName(jsonObj_CanBeNone) + elif commandPathLower.startswith("get-local-plugin-webcam-items"): + return self.GetPluginLocalWebcamSettingsItems(jsonObj_CanBeNone) + elif commandPathLower.startswith("set-local-plugin-webcam-items"): + return self.SetPluginLocalWebcamSettingsItems(jsonObj_CanBeNone) elif commandPathLower.startswith("pause"): return self.Pause(jsonObj_CanBeNone) elif commandPathLower.startswith("resume"): diff --git a/octoeverywhere/webcamhelper.py b/octoeverywhere/webcamhelper.py index fd9fa21..6f4c995 100644 --- a/octoeverywhere/webcamhelper.py +++ b/octoeverywhere/webcamhelper.py @@ -1,6 +1,8 @@ import logging import os import json +from typing import List + import urllib3 from .sentry import Sentry @@ -17,7 +19,7 @@ class WebcamSettingItem: # snapshotUrl OR streamUrl can be None if the values aren't available, but not both. # flipHBool & flipVBool & rotationInt must exist. # rotationInt must be 0, 90, 180, or 270 - def __init__(self, name:str = "", snapshotUrl:str = "", streamUrl:str = "", flipHBool:bool = False, flipVBool:bool = False, rotationInt:int = 0): + def __init__(self, name:str = "", snapshotUrl:str = "", streamUrl:str = "", flipHBool:bool = False, flipVBool:bool = False, rotationInt:int = 0, enabled:bool = True): self._name = "" self.Name = name self.SnapshotUrl = snapshotUrl @@ -25,6 +27,8 @@ def __init__(self, name:str = "", snapshotUrl:str = "", streamUrl:str = "", flip self.FlipH = flipHBool self.FlipV = flipVBool self.Rotation = rotationInt + # This is a special flag mostly used for the local plugin webcams to indicate they are no enabled. + self.Enabled = enabled @property @@ -62,6 +66,45 @@ def Validate(self, logger:logging.Logger) -> bool: return True + # Used to serialize the object to a dict that can be used with json. + # THESE PROPERTY NAMES CAN'T CHANGE, it's used for the API and it's used to serialize to disk. + def Serialize(self, includeUrls:bool = True) -> dict: + d = { + "Name": self.Name, + "FlipH": self.FlipH, + "FlipV": self.FlipV, + "Rotation": self.Rotation, + "Enabled": self.Enabled + } + if includeUrls: + d["SnapshotUrl"] = self.SnapshotUrl + d["StreamUrl"] = self.StreamUrl + return d + + + # Used to convert a dict back into a WebcamSettingItem object. + # Returns None if there's a failure + @staticmethod + def Deserialize(d:dict, logger:logging.Logger): + try: + name = d.get("Name") + snapshotUrl = d.get("SnapshotUrl") + streamUrl = d.get("StreamUrl") + flipH = d.get("FlipH") + flipV = d.get("FlipV") + rotation = d.get("Rotation") + enabled = d.get("Enabled") + if name is None or snapshotUrl is None or streamUrl is None or flipH is None or flipV is None or rotation is None or enabled is None: + raise Exception("Failed to deserialize WebcamSettingItem, missing values.") + i = WebcamSettingItem(str(name), str(snapshotUrl), str(streamUrl), bool(flipH), bool(flipV), int(rotation), bool(enabled)) + if i.Validate(logger) is False: + raise Exception("Failed to validate WebcamSettingItem.") + return i + except Exception as e: + Sentry.Exception("Failed to deserialize WebcamSettingItem", e) + return None + + # The point of this class is to abstract the logic that needs to be done to reliably get a webcam snapshot and stream from many types of # printer setups. The main entry point is GetSnapshot() which will try a number of ways to get a snapshot from whatever camera system is # setup. This includes USB based cameras, external IP based cameras, and OctoPrint instances that don't have a snapshot URL defined. @@ -95,9 +138,12 @@ def Get(): def __init__(self, logger:logging.Logger, webcamPlatformHelperInterface, pluginDataFolderPath:str): self.Logger = logger self.WebcamPlatformHelperInterface = webcamPlatformHelperInterface + + # Init local webcam settings stuffs. self.SettingsFilePath = os.path.join(pluginDataFolderPath, "webcam-settings.json") - self.DefaultCameraName = None - self._LoadDefaultCameraName() + self.DefaultCameraName:str = None + self.LocalPluginWebcamSettingsObjects:List[WebcamSettingItem] = [] + self._LoadPluginWebcamSettings() # Returns the snapshot URL from the settings. @@ -187,10 +233,13 @@ def GetWebcamStream(self, cameraIndex:int = None) -> OctoHttpRequest.Result: return self._AddOeWebcamTransformHeader(self._GetWebcamStreamInternal(cameraIndex), cameraIndex) - def _GetWebcamStreamInternal(self, cameraIndex:int) -> OctoHttpRequest.Result: - # Check if the platform helper has an override. If so, it is responsible for all of the stream getting logic. + def _GetWebcamStreamInternal(self, cameraIndex:int = None) -> OctoHttpRequest.Result: + # Check if the platform helper has an override. + # If so, it CAN BE responsible for all of the snapshot getting logic, but if returns None we should fallback to the default logic. if hasattr(self.WebcamPlatformHelperInterface, 'GetStream_Override'): - return self.WebcamPlatformHelperInterface.GetStream_Override(cameraIndex) + ret = self.WebcamPlatformHelperInterface.GetStream_Override(self._GetWebcamSettingObj(cameraIndex)) + if ret is not None: + return ret # Try to get the URL from the settings. webcamStreamUrl = self.GetWebcamStreamUrl(cameraIndex) @@ -218,10 +267,13 @@ def GetSnapshot(self, cameraIndex:int = None) -> OctoHttpRequest.Result: return self._AddOeWebcamTransformHeader(self._EnsureJpegHeaderInfo(self._GetSnapshotInternal(cameraIndex)), cameraIndex) - def _GetSnapshotInternal(self, cameraIndex:int) -> OctoHttpRequest.Result: - # Check if the platform helper has an override. If so, it is responsible for all of the snapshot getting logic. - if hasattr(self.WebcamPlatformHelperInterface, 'GetSnapshot_Override'): - return self.WebcamPlatformHelperInterface.GetSnapshot_Override(cameraIndex) + def _GetSnapshotInternal(self, cameraIndex:int = None) -> OctoHttpRequest.Result: + # Check if the platform helper has an override. + # If so, it CAN BE responsible for all of the snapshot getting logic, but if returns None we should fallback to the default logic. + if hasattr(self.WebcamPlatformHelperInterface, 'GetSnapshot_Override'): + ret = self.WebcamPlatformHelperInterface.GetSnapshot_Override(self._GetWebcamSettingObj(cameraIndex)) + if ret is not None: + return ret # First, try to get the snapshot using the string defined in settings. snapshotUrl = self.GetSnapshotUrl(cameraIndex) @@ -417,12 +469,23 @@ def _GetWebcamSettingObj(self, cameraIndex:int = None): # Returns the currently known list of webcams. # The order they are returned is the order the use sees them. # The default is usually the index 0. - def ListWebcams(self): + def ListWebcams(self) -> List[WebcamSettingItem]: try: - a = self.WebcamPlatformHelperInterface.GetWebcamConfig() - if a is None or len(a) == 0: + # Get the webcams from the platform. + ret = self.WebcamPlatformHelperInterface.GetWebcamConfig() + + # Check if there are any plugin local items to return. + # Note the cameras returned from ListWebcams() must always be first - the bambu logic depends on this! (see GetSnapshot_Override) + pluginLocalWebcamItems = self.GetPluginLocalWebcamList() + if pluginLocalWebcamItems is not None and len(pluginLocalWebcamItems) > 0: + if ret is None: + ret = [] + ret.extend(pluginLocalWebcamItems) + + # Ensure we got something + if ret is None or len(ret) == 0: return None - return a + return ret except Exception as e: Sentry.Exception("WebcamHelper ListWebcams exception.", e) return None @@ -635,37 +698,32 @@ def FixMissingSlashInWebcamUrlIfNeeded(logger:logging.Logger, webcamUrl:str) -> # - # Default camera name logic. + # Plugin Webcam Logic. # The default camera is always set and stored as the name, since the camera index can change over time. # But it's always gotten as the index of the current list of cameras. # + # The plugin can also have a local list of cameras it will add to the main get cameras result, so that users can + # setup their own cameras in the plugin settings on the website. This is used for systems like the Bambu, where there's no other UI for settings. + # # Sets the default camera name and writes it to the settings file. def SetDefaultCameraName(self, name:str) -> None: name = name.lower() self.DefaultCameraName = name - try: - settings = { - "DefaultWebcamName" : self.DefaultCameraName - } - with open(self.SettingsFilePath, encoding="utf-8", mode="w") as f: - f.write(json.dumps(settings)) - except Exception as e: - self.Logger.error("SetDefaultCameraName failed "+str(e)) + self._SavePluginWebcamSettings() # Returns the default camera index. This will always return an int. # If there is not a default currently set, this returns the WebcamHelper.c_DefaultWebcamIndex, which is index 0. - def GetDefaultCameraIndex(self, webcamItemList) -> int: + def GetDefaultCameraIndex(self, webcamItemList:List[WebcamSettingItem]) -> int: # If there is no name currently, the default is 0. if self.DefaultCameraName is None: return WebcamHelper.c_DefaultWebcamIndex # Try to find the name that was last set. - defaultCameraNameLower = self.DefaultCameraName.lower() count = 0 for i in webcamItemList: - if i.Name == defaultCameraNameLower: + if i.Name.lower() == self.DefaultCameraName: return count count += 1 @@ -673,11 +731,64 @@ def GetDefaultCameraIndex(self, webcamItemList) -> int: return WebcamHelper.c_DefaultWebcamIndex + # Returns a list of any plugin local webcam settings objects. + # These objects will be merged into the main list of webcams settings objects + def GetPluginLocalWebcamList(self, returnDisabledItems:bool = False) -> List[WebcamSettingItem]: + # If there's nothing return or we are returning everything, return the list. + if len(self.LocalPluginWebcamSettingsObjects) == 0 or returnDisabledItems: + return self.LocalPluginWebcamSettingsObjects + # Otherwise return the list of enabled objects. + ret = [] + for i in self.LocalPluginWebcamSettingsObjects: + if i.Enabled: + ret.append(i) + return ret + + + # Sets the local plugin webcam settings objects. + def SetPluginLocalWebcamList(self, newList:List[WebcamSettingItem]) -> bool: + # Validate the new list of webcam items. + for i in newList: + if i.Validate(self.Logger) is False: + self.Logger.warn(f"SetPluginLocalWebcamList failed to validate the webcam settings object. {i.Name}") + return False + + # Set the new list. + self.LocalPluginWebcamSettingsObjects = newList + + # Save the settings + return self._SavePluginWebcamSettings() + + + # Saves the currently set plugin webcam settings to the settings file. + def _SavePluginWebcamSettings(self) -> bool: + try: + # Convert the local webcam settings objects to dicts + localWebcamSettingsDict = [] + for i in self.LocalPluginWebcamSettingsObjects: + localWebcamSettingsDict.append(i.Serialize()) + + # Create the settings object + settings = { + "DefaultWebcamName" : self.DefaultCameraName, + "LocalPluginWebcamSettings": localWebcamSettingsDict + } + + # Save + with open(self.SettingsFilePath, encoding="utf-8", mode="w") as f: + f.write(json.dumps(settings)) + return True + except Exception as e: + self.Logger.error("SetDefaultCameraName failed "+str(e)) + return False + + # Loads the current name from our settings file. - def _LoadDefaultCameraName(self) -> None: + def _LoadPluginWebcamSettings(self) -> None: try: - # Default the setting. + # Default the settings. self.DefaultCameraName = None + self.LocalPluginWebcamSettingsObjects = [] # First check if there's a file. if os.path.exists(self.SettingsFilePath) is False: @@ -687,11 +798,20 @@ def _LoadDefaultCameraName(self) -> None: with open(self.SettingsFilePath, encoding="utf-8") as f: data = json.load(f) - name = data["DefaultWebcamName"] - if name is None or len(name) == 0: - return - self.DefaultCameraName = name - self.Logger.info(f"Webcam settings loaded. Default camera name: {self.DefaultCameraName}") + # Get the default webcam name. + name:str = data.get("DefaultWebcamName", None) + if name is not None and len(name) > 0: + self.DefaultCameraName = name.lower() + + # Set the local plugin webcam setting items. + items:List[WebcamSettingItem] = data.get("LocalPluginWebcamSettings", None) + if items is not None and len(items) > 0: + for i in items: + wsi = WebcamSettingItem.Deserialize(i, self.Logger) + if wsi is not None: + self.LocalPluginWebcamSettingsObjects.append(wsi) + + self.Logger.info(f"Webcam settings loaded. Default camera name: {self.DefaultCameraName}, Local Webcam Settings Items: {len(self.LocalPluginWebcamSettingsObjects)}") except Exception as e: self.Logger.error("_LoadDefaultCameraName failed "+str(e)) diff --git a/setup.py b/setup.py index 69ed4a9..5e4a02c 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this is also parsed by the moonraker module to pull the version, so the string and format must remain the same! -plugin_version = "3.2.8" +plugin_version = "3.3.0" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From cba654901d7fa7eadfbdb689637f6936b9c5919d Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Wed, 1 May 2024 20:16:38 -0700 Subject: [PATCH 079/328] Adding support for RTSP webcam feeds for all platforms! --- .vscode/settings.json | 2 + bambu_octoeverywhere/bambuhost.py | 6 +- bambu_octoeverywhere/bambuwebcamhelper.py | 207 ++++++------- moonraker_octoeverywhere/moonrakerhost.py | 2 +- .../moonrakerwebcamhelper.py | 13 +- .../WebStream/octowebstreamhttphelper.py | 2 +- octoeverywhere/Webcam/__init__.py | 0 .../Webcam}/quickcam.py | 280 +++++++++++++----- octoeverywhere/{ => Webcam}/webcamhelper.py | 186 ++---------- octoeverywhere/Webcam/webcamsettingitem.py | 117 ++++++++ octoeverywhere/Webcam/webcamstreaminstance.py | 80 +++++ octoeverywhere/commandhandler.py | 3 +- octoeverywhere/notificationshandler.py | 2 +- octoprint_octoeverywhere/__init__.py | 2 +- octoprint_octoeverywhere/__main__.py | 2 +- .../octoprintwebcamhelper.py | 14 +- py_installer/Ffmpeg.py | 13 +- py_installer/Installer.py | 6 +- setup.py | 2 +- 19 files changed, 572 insertions(+), 367 deletions(-) create mode 100644 octoeverywhere/Webcam/__init__.py rename {bambu_octoeverywhere => octoeverywhere/Webcam}/quickcam.py (69%) rename octoeverywhere/{ => Webcam}/webcamhelper.py (83%) create mode 100644 octoeverywhere/Webcam/webcamsettingitem.py create mode 100644 octoeverywhere/Webcam/webcamstreaminstance.py diff --git a/.vscode/settings.json b/.vscode/settings.json index a252019..2bc68e9 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -223,7 +223,9 @@ "warmingup", "webassets", "webcamhelper", + "webcamsettingitem", "webcamstream", + "webcamstreaminstance", "webfont", "webfonts", "webrequestresponsehandler", diff --git a/bambu_octoeverywhere/bambuhost.py b/bambu_octoeverywhere/bambuhost.py index 0102867..0c47cb2 100644 --- a/bambu_octoeverywhere/bambuhost.py +++ b/bambu_octoeverywhere/bambuhost.py @@ -6,7 +6,7 @@ from octoeverywhere.telemetry import Telemetry from octoeverywhere.hostcommon import HostCommon from octoeverywhere.printinfo import PrintInfoManager -from octoeverywhere.webcamhelper import WebcamHelper +from octoeverywhere.Webcam.webcamhelper import WebcamHelper from octoeverywhere.octopingpong import OctoPingPong from octoeverywhere.commandhandler import CommandHandler from octoeverywhere.octoeverywhereimpl import OctoEverywhere @@ -24,7 +24,6 @@ from .bambuwebcamhelper import BambuWebcamHelper from .bambucommandhandler import BambuCommandHandler from .bambustatetranslater import BambuStateTranslator -from .quickcam import QuickCam # This file is the main host for the bambu service. class BambuHost: @@ -110,8 +109,7 @@ def RunBlocking(self, configPath, localStorageDir, repoRoot, devConfig_CanBeNone if DevLocalServerAddress_CanBeNone is not None: OctoPingPong.Get().DisablePrimaryOverride() - # Setup the webcam helper and QuickCam - QuickCam.Init(self.Logger, self.Config) + # Setup the webcam helper webcamHelper = BambuWebcamHelper(self.Logger, self.Config) WebcamHelper.Init(self.Logger, webcamHelper, localStorageDir) diff --git a/bambu_octoeverywhere/bambuwebcamhelper.py b/bambu_octoeverywhere/bambuwebcamhelper.py index 2eb463c..755b042 100644 --- a/bambu_octoeverywhere/bambuwebcamhelper.py +++ b/bambu_octoeverywhere/bambuwebcamhelper.py @@ -1,27 +1,23 @@ import logging import time -import threading - -from octoeverywhere.webcamhelper import WebcamSettingItem -from octoeverywhere.octohttprequest import OctoHttpRequest from linux_host.config import Config -from .quickcam import QuickCam +from octoeverywhere.sentry import Sentry +from octoeverywhere.Webcam.webcamsettingitem import WebcamSettingItem + +from .bambuclient import BambuClient # This class implements the webcam platform helper interface for bambu. class BambuWebcamHelper(): - # These don't really matter, but we define them to keep them consistent - c_SpecialMockSnapshotPath = "bambu-special-snapshot" - c_SpecialMockStreamPath = "bambu-special-stream" - c_OeStreamBoundaryString = "oestreamboundary" - def __init__(self, logger:logging.Logger, config:Config) -> None: self.Logger = logger self.Config = config + self.LastUrlUpdateTimeSec:float = 0.0 + self.CachedStreamingUrl = None # !! Interface Function !! @@ -30,115 +26,94 @@ def __init__(self, logger:logging.Logger, config:Config) -> None: # The order the webcams are returned is the order the user will see in any selection UIs. # Returns None on failure. def GetWebcamConfig(self): - # Bambu has a special webcam setup where there's only one cam and we need to get in a special way, so we return this one default webcam object. + # Bambu has a special webcam setup where there's only one cam, and the streaming system is either RTSP or Websocket based. # We do support plugin local webcam items, which are webcams the user can setup from the website and are external webcams. # Note! This webcam name is shown on to the user in the UI, so it should be a good name that indicates this is a Bambu built in webcam. # Also, if the name changes, the default printer index might also change. - return [WebcamSettingItem("Bambu Cam", BambuWebcamHelper.c_SpecialMockSnapshotPath, BambuWebcamHelper.c_SpecialMockStreamPath, False, False, 0)] - - - # !! Optional Interface Function !! - # If defined, this function must handle ALL snapshot requests for the platform. - # - # On failure or if this camera doesn't need the override, return None - # On success, this will return a valid OctoHttpRequest that's fully filled out. - # The snapshot will always already be fully read, and will be FullBodyBuffer var. - def GetSnapshot_Override(self, webcamSettingsItem:WebcamSettingItem): - # Detect if this request is for our special snapshot logic, if not, return None to use the default webcam handling. - if webcamSettingsItem.SnapshotUrl != BambuWebcamHelper.c_SpecialMockSnapshotPath: - return None - - # Try to get a snapshot from our QuickCam system. - img = QuickCam.Get().GetCurrentImage() - if img is None: - return None - - # If we get an image, return it! - headers = { - "Content-Type": "image/jpeg" - } - return OctoHttpRequest.Result(200, headers, BambuWebcamHelper.c_SpecialMockSnapshotPath, False, fullBodyBuffer=img) - - - # !! Optional Interface Function !! - # If defined, this function must handle ALL stream requests for the platform. - # - # On failure, return None - # On success, this will return a valid OctoHttpRequest that's fully filled out. - # This must return an OctoHttpRequest object with a custom body read stream. - def GetStream_Override(self, webcamSettingsItem:WebcamSettingItem): - # Detect if this request is for our special snapshot logic, if not, return None to use the default webcam handling. - if webcamSettingsItem.StreamUrl != BambuWebcamHelper.c_SpecialMockStreamPath: - return None - - # We must create a new instance of this class per stream to ensure all of the vars stay in it's context - # and the streams are cleaned up properly. - sm = StreamInstance(self.Logger) - return sm.StartWebRequest() - - -# Stream Instance is a class that is created per web stream to handle streaming QuickCam images into the http stream. -# It must be created per http request so it can manage it's own local vars. -class StreamInstance: - def __init__(self, logger:logging.Logger) -> None: - self.Logger = logger - self.IsFirstSend = True - self.StreamOpenTimeSec = time.time() - self.ImageReadyEvent = threading.Event() - self.AwaitingImage:bytearray = None - - - def StartWebRequest(self) -> OctoHttpRequest.Result: - # First, try to get a snapshot. This will determine if we are able to get a stream or not. - # If we can't start the stream, then we don't return success. - # We will also use this first image to start the stream, to get it going ASAP. - self.AwaitingImage = QuickCam.Get().GetCurrentImage() - if self.AwaitingImage is None: - return None + return [WebcamSettingItem("Bambu Cam", None, self._GetStreamingUrl(), False, False, 0)] - # Note! We must be sure to call DetachImageStreamCallback to remove this stream callback! - QuickCam.Get().AttachImageStreamCallback(self._NewImageCallback) - # We must set the content type so that the web browser knows what kind of stream to expect. - headers = { - "content-type": f"multipart/x-mixed-replace; boundary={BambuWebcamHelper.c_OeStreamBoundaryString}", - } - # Return a result object with out callbacks setup for the stream body. - return OctoHttpRequest.Result(200, headers, BambuWebcamHelper.c_SpecialMockStreamPath, False, customBodyStreamCallback=self._CustomBodyStreamRead, customBodyStreamClosedCallback=self._CustomBodyStreamClosed) - - - # Define the callback we will get from QuickCam when there's a new image ready for us to send. - def _NewImageCallback(self, imgBuffer:bytearray): - self.AwaitingImage = imgBuffer - self.ImageReadyEvent.set() - - - # Define a callback for our http body reading system to call when it needs data. - def _CustomBodyStreamRead(self) -> bytearray: + # !! Interface Function !! + # This function is called to determine if a QuickCam stream should keep running or not. + # The idea is since a QuickCam stream can take longer to start, for example, the Bambu Websocket stream on sends 1FPS, + # we can keep the stream running while the print is running to lower the latency of getting images. + # Most most platforms, this should return true if the print is running or paused, otherwise false. + # Also consider something like Gadget, it takes pictures every 20-40 seconds, so the stream will be started frequently if it's not already running. + def ShouldQuickCamStreamKeepRunning(self) -> bool: + # For Bambu, we want to keep the stream running if the printer is printing. + state = BambuClient.Get().GetState() + if state is None: + return False + return state.IsPrinting(True) + + + # Returns the current URL that should be used for snapshots and streaming. + def _GetStreamingUrl(self) -> str: + # We cache the urls for a little bit once they are generated, so we don't have to re-created them every time + # But we do want to refresh them occasionally, so if the access code or IP changes, we update it. + self._UpdateUrlsIfNeeded() + + # Ensure we got something, if not, warn about it. + if self.CachedStreamingUrl is None: + self.Logger.error("BambuWebcamHelper failed to get streaming URL, thus we can't stream.") + return "none" + return self.CachedStreamingUrl + + + # This needs to be thread safe, but we don't use any locks. It's find if multiple threads get the info at the same time. + def _UpdateUrlsIfNeeded(self) -> None: + # Test if we need to update or use the cached values. + if self.CachedStreamingUrl is not None and len(self.CachedStreamingUrl) > 0 and time.time() - self.LastUrlUpdateTimeSec < 30.0: + return + + # Before we can return the webcam config, we need to know what kind of printer this is. + # TODO - Right now it seems the X1 doesn't send back version info on start or with the version command, + # so we use the existence of the RTSP URL to determine what we can do. + # Ideally we would use the printer version in the future. + rtspUrl = None + stateGetAttempt = 0 while True: - # See if we can capture an image. There might already be a new image we don't even have to wait for. - capturedImage = self.AwaitingImage - if capturedImage is not None: - # If so, clear the awaiting image and reset the event. - self.AwaitingImage = None - self.ImageReadyEvent.clear() - - # Build the buffer to send - header = f"--{BambuWebcamHelper.c_OeStreamBoundaryString}\r\nContent-Type: image/jpeg\r\nContent-Length: {len(capturedImage)}\r\n\r\n" - imageChunkBuffer = header.encode('utf-8') + capturedImage + b"\r\n" + header.encode('utf-8') + capturedImage + b"\r\n" - - # TODO - I don't know why, but chrome seems to delay the rendering of the image until it gets two? - # This could be something in the pipeline not flushing correctly, or other things. But for now, on the first send we double the image to make it render instantly. - if self.IsFirstSend: - imageChunkBuffer = imageChunkBuffer + imageChunkBuffer - self.IsFirstSend = False - self.Logger.info(f"QuickCam took {time.time()-self.StreamOpenTimeSec} seconds from stream open to first image sent.") - return imageChunkBuffer - # If we didn't get an image, wait on the event for a new one. - self.ImageReadyEvent.wait() - - - # Define a callback for when the http stream is closed. - def _CustomBodyStreamClosed(self) -> None: - # It's important this is called so the stream will be detached! - QuickCam.Get().DetachImageStreamCallback(self._NewImageCallback) + stateGetAttempt += 1 + # Wait until the object exists + state = BambuClient.Get().GetState() + if state is not None: + # When the state object is not None, we know we got the state sync. + # Now we can check if there's a RTSP url or not, which will indicate what kind of stream we need to use. + rtspUrl = state.rtsp_url + break + + # If we didn't get the state after a few attempts, we give up and default to the websocket stream. + if stateGetAttempt > 5: + self.Logger.warn(f"BambuWebcamHelper wasn't able to get the printer state after {stateGetAttempt} attempts") + break + + # Sleep for a bit before trying again. + time.sleep(2.0) + + # Get the access code and ip from the config file, so we always get the latest. + # The BambuClient class will update the value in the config if the IP address of the printer changes, which can happen while we are running. + accessCode = self.Config.GetStr(Config.SectionBambu, Config.BambuAccessToken, None) + ipOrHostname = self.Config.GetStr(Config.SectionCompanion, Config.CompanionKeyIpOrHostname, None) + if accessCode is None or ipOrHostname is None: + self.Logger.error("BambuWebcamHelper failed to get a ip or access code from the config, thus we can't stream.") + return + + # If there is a RTSP URL, we know this printer uses the RTSP protocol to stream the webcam. + if rtspUrl is not None and len(rtspUrl) > 0: + # Use the URL the X1 sent us, but inject the auth into it. + protocolEnd = rtspUrl.find("://") + if protocolEnd != -1: + protocolEnd += 3 + self.CachedStreamingUrl = rtspUrl[:protocolEnd] + f"bblp:{accessCode}@" + rtspUrl[protocolEnd:] + # We should be able to find the IP in the URL, warn if not. + if self.CachedStreamingUrl.find(ipOrHostname) == -1: + Sentry.LogError(f"BambuWebcamHelper didn't find the currently known IP of the printer in the RTSP URL returned from the printer. Printer URL:{rtspUrl} Known IP:{ipOrHostname}") + else: + self.Logger.error(f"BambuWebcamHelper failed to parse the return rtsp URL from the printer, using our own. {rtspUrl}") + self.CachedStreamingUrl = f"rtsps://bblp:{accessCode}@{ipOrHostname}:322/streaming/live/1" + else: + # If there is no RTSP URL, we assume the printer uses the websocket based cam streaming. + self.CachedStreamingUrl = f"ws://bblp:{accessCode}@{ipOrHostname}:6000" + + # Set the time we updated the cached values. + self.LastUrlUpdateTimeSec = time.time() diff --git a/moonraker_octoeverywhere/moonrakerhost.py b/moonraker_octoeverywhere/moonrakerhost.py index 0cea94b..74b203e 100644 --- a/moonraker_octoeverywhere/moonrakerhost.py +++ b/moonraker_octoeverywhere/moonrakerhost.py @@ -6,7 +6,7 @@ from octoeverywhere.telemetry import Telemetry from octoeverywhere.hostcommon import HostCommon from octoeverywhere.octopingpong import OctoPingPong -from octoeverywhere.webcamhelper import WebcamHelper +from octoeverywhere.Webcam.webcamhelper import WebcamHelper from octoeverywhere.printinfo import PrintInfoManager from octoeverywhere.commandhandler import CommandHandler from octoeverywhere.octoeverywhereimpl import OctoEverywhere diff --git a/moonraker_octoeverywhere/moonrakerwebcamhelper.py b/moonraker_octoeverywhere/moonrakerwebcamhelper.py index 5436b82..5e688a2 100644 --- a/moonraker_octoeverywhere/moonrakerwebcamhelper.py +++ b/moonraker_octoeverywhere/moonrakerwebcamhelper.py @@ -6,7 +6,7 @@ import requests from octoeverywhere.sentry import Sentry -from octoeverywhere.webcamhelper import WebcamSettingItem, WebcamHelper +from octoeverywhere.Webcam.webcamhelper import WebcamSettingItem, WebcamHelper from linux_host.config import Config @@ -111,6 +111,17 @@ def GetWebcamConfig(self): ] + # !! Interface Function !! + # This function is called to determine if a QuickCam stream should keep running or not. + # The idea is since a QuickCam stream can take longer to start, for example, the Bambu Websocket stream on sends 1FPS, + # we can keep the stream running while the print is running to lower the latency of getting images. + # Most most platforms, this should return true if the print is running or paused, otherwise false. + # Also consider something like Gadget, it takes pictures every 20-40 seconds, so the stream will be started frequently if it's not already running. + def ShouldQuickCamStreamKeepRunning(self) -> bool: + # TODO - Ideally this should return true if we are printing. + return False + + # Wakes up the auto settings worker. # Called by moonrakerclient when the websocket is connected, to ensure we pull settings on moonraker connections. def KickOffWebcamSettingsUpdate(self, forceUpdate = False): diff --git a/octoeverywhere/WebStream/octowebstreamhttphelper.py b/octoeverywhere/WebStream/octowebstreamhttphelper.py index 2595f60..d1af95e 100644 --- a/octoeverywhere/WebStream/octowebstreamhttphelper.py +++ b/octoeverywhere/WebStream/octowebstreamhttphelper.py @@ -11,7 +11,7 @@ from .octoheaderimpl import BaseProtocol from ..octohttprequest import OctoHttpRequest from ..octostreammsgbuilder import OctoStreamMsgBuilder -from ..webcamhelper import WebcamHelper +from ..Webcam.webcamhelper import WebcamHelper from ..commandhandler import CommandHandler from ..sentry import Sentry from ..compat import Compat diff --git a/octoeverywhere/Webcam/__init__.py b/octoeverywhere/Webcam/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bambu_octoeverywhere/quickcam.py b/octoeverywhere/Webcam/quickcam.py similarity index 69% rename from bambu_octoeverywhere/quickcam.py rename to octoeverywhere/Webcam/quickcam.py index f2a8d7e..3a1e66f 100644 --- a/bambu_octoeverywhere/quickcam.py +++ b/octoeverywhere/Webcam/quickcam.py @@ -11,33 +11,121 @@ from octoeverywhere.sentry import Sentry -from linux_host.config import Config +from ..octohttprequest import OctoHttpRequest +from .webcamsettingitem import WebcamSettingItem +from .webcamstreaminstance import WebcamStreamInstance -from .bambuclient import BambuClient -# The goal of this class is to handle webcam streaming and snapshots. The idea is since we need to establish a socket and stream to even get snapshots, -# rather than doing it over and over, we will keep the stream alive for a short period of time and take snapshots, so when the user wants them, they are ready. -class QuickCam: +# Indicates the stream type for the QuickCam class. +# The NotSupported means that the URL parsed isn't supported by QuickCam. +class QuickCamStreamTypes: + NotSupported = 0 + RTSP = 1 + WebSocket = 2 - # The amount of time the capture thread will stay connected before it will close. - # Whenever an image is accessed, the time is reset. - c_CaptureThreadTimeoutSec = 60 + +# This is a helper class that manages active instances of QuickCam and allows all requests for the same URL to share a common stream. +class QuickCamManager: _Instance = None @staticmethod - def Init(logger:logging.Logger, config:Config): - QuickCam._Instance = QuickCam(logger, config) + def Init(logger:logging.Logger, webcamPlatformHelperInterface): + QuickCamManager._Instance = QuickCamManager(logger, webcamPlatformHelperInterface) @staticmethod def Get(): - return QuickCam._Instance + return QuickCamManager._Instance - def __init__(self, logger:logging.Logger, config:Config ) -> None: + def __init__(self, logger:logging.Logger, webcamPlatformHelperInterface) -> None: self.Logger = logger - self.Config = config + self.WebcamPlatformHelperInterface = webcamPlatformHelperInterface + self.QuickCamMap = {} + self.QuickCamMapLock = threading.Lock() + + + # Given the webcam settings item, this will check if the settings item needs to use any of the supported QuickCam streaming capture methods. + # On success, this will return a complete OctoHttpResult object, otherwise None + def TryToGetSnapshot(self, webcamSettingsItem:WebcamSettingItem): + # To know if we need to use Quick cam, we check the protocols. + # We check both the snapshot and streaming URL, since we can get a snapshot from either + url = webcamSettingsItem.SnapshotUrl + t = QuickCam.GetStreamTypeFromUrl(url) + if t == QuickCamStreamTypes.NotSupported: + url = webcamSettingsItem.StreamUrl + t = QuickCam.GetStreamTypeFromUrl(url) + if t == QuickCamStreamTypes.NotSupported: + return None + + # If we got here, then this is a URL type we need to use QuickCam for. + # Get the QuickCam instance for this URL. + qc = self._GetOrCreate(url) + + # Try to get a snapshot. + img = qc.GetCurrentImage() + if img is None: + return None + + # If we get an image, return it! + headers = { + "Content-Type": "image/jpeg" + } + return OctoHttpRequest.Result(200, headers, url, False, fullBodyBuffer=img) + + + # Given the webcam settings item, this will check if the settings item needs to use any of the supported QuickCam streaming capture methods. + # On failure, return None + # On success, this will return a valid OctoHttpRequest that's fully filled out. + # This must return an OctoHttpRequest object with a custom body read stream. + def TryGetStream(self, webcamSettingsItem:WebcamSettingItem): + # To know if we need to use Quick cam, we check the protocols. + # We check both the snapshot and streaming URL, since we can get a snapshot from either + url = webcamSettingsItem.StreamUrl + t = QuickCam.GetStreamTypeFromUrl(url) + if t == QuickCamStreamTypes.NotSupported: + return None + + # If we got here, then this is a url type we need to use QuickCam for. + # Get the QuickCam instance for this URL. + qc = self._GetOrCreate(url) + + # We must create a new instance of this class per stream to ensure all of the vars stay in it's context and the streams are cleaned up properly. + # Create the stream instance and start the web request. + sm = WebcamStreamInstance(self.Logger, qc) + return sm.StartWebRequest() + + + # Returns a QuickCam instances for this url. If auth is required, the auth should be added to the URL in the http:// style. Ex rtsp://username:password@hostname... + # The QuickCam class will be shared across multiple instances, it's thread safe. + def _GetOrCreate(self, url:str): + with self.QuickCamMapLock: + # If it already exists, get it and return it. + qc = self.QuickCamMap.get(url, None) + if qc is not None: + return qc + + # Otherwise create it. + qc = QuickCam(self.Logger, url, self.WebcamPlatformHelperInterface) + self.QuickCamMap[url] = qc + return qc + + +# This class handles webcam streaming from different streaming endpoints that aren't http. +# Right now it supports RTSP camera feeds and the Bambu Websocket based streaming protocol. +class QuickCam: + + # The amount of time the capture thread will stay connected before it will close. + # Whenever an image is accessed, the time is reset. + c_CaptureThreadTimeoutSec = 60 + + + def __init__(self, logger:logging.Logger, url:str, webcamPlatformHelperInterface) -> None: + self.Logger = logger + self.WebcamPlatformHelperInterface = webcamPlatformHelperInterface + self.Type = QuickCam.GetStreamTypeFromUrl(url) + self.Url = url self.Lock = threading.Lock() self.ImageReady = threading.Event() self.IsCaptureThreadRunning = False @@ -47,6 +135,22 @@ def __init__(self, logger:logging.Logger, config:Config ) -> None: self.ImageStreamCallbackLock = threading.Lock() + # Given a URL, this function returns the quick cam type that will be used and if it's supported. + @staticmethod + def GetStreamTypeFromUrl(url:str) -> QuickCamStreamTypes: + # Ensure there's something to parse + if url is None or len(url) == 0: + return QuickCamStreamTypes.NotSupported + url = url.lower() + # Check if the URL is RTSP + if url.startswith("rtsps://") or url.startswith("rtsp://"): + return QuickCamStreamTypes.RTSP + # Or if the URL is a websocket. We will assume it's the Bambu websocket protocol if so. + if url.startswith("ws://") or url.startswith("wss://"): + return QuickCamStreamTypes.WebSocket + return QuickCamStreamTypes.NotSupported + + # Tries to get the current image from the printer and returns it as a raw jpeg. # This will return None if it fails. def GetCurrentImage(self) -> bytearray: @@ -78,12 +182,17 @@ def GetCurrentImage(self) -> bytearray: # Used to attach a new stream handler to receive callbacks when an image is ready. # Note a call to detach must be called as well! def AttachImageStreamCallback(self, callback): + # Add our callback to the list. with self.ImageStreamCallbackLock: self.ImageStreamCallbacks.append(callback) + # Ensure that the capture thread is running. + self._ensureCaptureThreadRunning() + # Used to detach a new stream handler to receive callbacks when an image is ready. def DetachImageStreamCallback(self, callback): + # Remove our callback. with self.ImageStreamCallbackLock: self.ImageStreamCallbacks.remove(callback) @@ -118,32 +227,6 @@ def _ensureCaptureThreadRunning(self): # Does the image image capture work. def _captureThread(self): try: - # Get the access code and the host name. - accessCode = self.Config.GetStr(Config.SectionBambu, Config.BambuAccessToken, None) - ipOrHostname = self.Config.GetStr(Config.SectionCompanion, Config.CompanionKeyIpOrHostname, None) - if accessCode is None or ipOrHostname is None: - raise Exception("QuickCam doesn't have a access code or ip to use.") - - # TODO - Right now it seems the X1 doesn't send back version info on start or with the version command - # So we use the existence of the RTSP URL to determine what we can do. - # Ideally we would use the printer version in the future. - rtspUrl = None - verAttempt = 0 - while True: - verAttempt += 1 - state = BambuClient.Get().GetState() - # Wait until the object exists - if state is not None: - rtspUrl = state.rtsp_url - break - # If we can't get it return, and then the quick cam thread will be started again - # When there's another request. - if verAttempt > 5: - self.Logger.warn(f"QuickCam wasn't able to get the printer state after {verAttempt} attempts") - return - # Sleep for a bit. - time.sleep(2.0) - # We allow a few attempts, so if there are any connection issues or errors we buffer them out. attempts = 0 while attempts < 5: @@ -151,22 +234,21 @@ def _captureThread(self): # Create the camera implementation we need for this device. camImpl = None - # Since we have to use the URL.... - # IF the URL is empty, it's an X1 with LAN streaming disabled. - # If the URL has an address, it's an X1 with LAN streaming. - # If it's None, it's a P1, A1, or another printer with no RTSP. - if rtspUrl is not None: - self.Logger.debug(f"Bambu RTSP URL is: `{rtspUrl}`") + if self.Type == QuickCamStreamTypes.RTSP: + self.Logger.debug(f"QuickCam capture thread started for RTSP. {self.Url}") camImpl = QuickCam_RTSP(self.Logger) - else: - # Default to the websocket impl, since it's used on the most printers. + elif self.Type == QuickCamStreamTypes.WebSocket: + self.Logger.debug(f"QuickCam capture thread started for Websocket. {self.Url}") camImpl = QuickCam_WebSocket(self.Logger) + else: + self.Logger.error("Quick cam tried to start a capture thread with an unsupported type. "+self.Url) + return # Wrap the usage into a with, so the connection is always cleaned up with camImpl: try: # Connect to the server. - camImpl.Connect(ipOrHostname, accessCode) + camImpl.Connect(self.Url) # Begin the capture loop. while True: @@ -174,15 +256,14 @@ def _captureThread(self): # This can return None, which means we should just check the time and spin. img = camImpl.GetImage() - # Check if we are done running, if so, leave + # Check if we are done running if time.time() - self.LastImageRequestTimeSec > QuickCam.c_CaptureThreadTimeoutSec: - # TODO - For now, we don't stop the webcam loop while the printer is printing. - # This allows for notifications, Gadget, snapshots, streams, and such to load super easily. - # We need to measure the load on this though. - state = BambuClient.Get().GetState() - if state is None or not state.IsPrinting(True): - # This will invoke the finally clause and leave. + # We are past our max time between image requests, ask the platform if we should keep running or not. + # The decision is platform specific, but usually if a print is running we want to keep this stream alive for lower latency snapshots. + if self.WebcamPlatformHelperInterface.ShouldQuickCamStreamKeepRunning() is False: return + # If we don't want to be done, set the last image request time to now, so we don't constantly query the platform. + self.LastImageRequestTimeSec = time.time() # Set the image if we got one. if img is not None: @@ -208,6 +289,29 @@ def _captureThread(self): self.Logger.info("QuickCam capture thread exit.") + # Given a URL in the protocol://username:password@example.com/ format, returns the username and password + # This will always return the URL. If a username and password were found, the will be removed. + # This will return the username and password if found, otherwise None + @staticmethod + def ParseOurUsernameAndPasswordFromUrlIfExists(url:str): + # Parse the username and password from the URL if it exists. + userName = None + password = None + if url.find("://") != -1: + hostnameAndPath = url.split("://")[1] + if hostnameAndPath.find("@") != -1: + userNameAndPassword = hostnameAndPath.split("@")[0] + if userNameAndPassword.find(":") -1: + userName, password = userNameAndPassword.split(":") + else: + userName = userNameAndPassword + # Remove the username and password from the URL + protocolEnd = url.find("://") + 3 + atSign = url.find("@") + url = url[:protocolEnd] + url[atSign+1:] + return url, userName, password + + # Implements the websocket camera version for the P1 and A1 series printers. class QuickCam_WebSocket: @@ -226,34 +330,54 @@ def __init__(self, logger:logging.Logger): # ~~ Interface Function ~~ # Connects to the server. # This will throw an exception if it fails. - def Connect(self, ipOrHostname:str, accessCode:str) -> None: - # Build the auth packet - authData = bytearray() - authData += struct.pack(" None: + + # Parse the username and password from the URL if it exists. + # This will always return the URL, striped of the username and password if found. + (url, userName, password) = QuickCam.ParseOurUsernameAndPasswordFromUrlIfExists(url) + + # Build the auth packet if needed + authData = None + if userName is not None and password is not None: + authData= bytearray() + authData += struct.pack(" None: - # TODO get the address from the bambu state object + def Connect(self, url:str) -> None: # We set the logging level of ffmpeg depending on our logging level # The logs are written to stderr even if they aren't errors, which is nice, so # we can capture them on timeouts. logLevel = "trace" if self.Logger.isEnabledFor(logging.DEBUG) else "warning" + # For auth, if there's a username and password it will already be in the URL in the http:// basic auth style, + # So there's nothing else we need to do. + # Notes # We use 15 fps because it's a good trade off of fps and cpu perf hits # It also decreases the bandwidth needed, which helps on mobile @@ -371,7 +497,7 @@ def Connect(self, ipOrHostname:str, accessCode:str) -> None: "-loglevel", logLevel, "-rtsp_transport", "udp", "-use_wallclock_as_timestamps", "1", - "-i", f"rtsps://bblp:{accessCode}@{ipOrHostname}:322/streaming/live/1", + "-i", url, "-filter:v", "fps=15", "-movflags", "+faststart", "-f", "image2pipe", "-" diff --git a/octoeverywhere/webcamhelper.py b/octoeverywhere/Webcam/webcamhelper.py similarity index 83% rename from octoeverywhere/webcamhelper.py rename to octoeverywhere/Webcam/webcamhelper.py index 6f4c995..731ffe0 100644 --- a/octoeverywhere/webcamhelper.py +++ b/octoeverywhere/Webcam/webcamhelper.py @@ -5,105 +5,10 @@ import urllib3 -from .sentry import Sentry -from .octohttprequest import OctoHttpRequest - -# -# A platform agnostic definition of a webcam stream. -# -class WebcamSettingItem: - - # The snapshotUrl and streamUrl can be relative or absolute. - # - # name must exist. - # snapshotUrl OR streamUrl can be None if the values aren't available, but not both. - # flipHBool & flipVBool & rotationInt must exist. - # rotationInt must be 0, 90, 180, or 270 - def __init__(self, name:str = "", snapshotUrl:str = "", streamUrl:str = "", flipHBool:bool = False, flipVBool:bool = False, rotationInt:int = 0, enabled:bool = True): - self._name = "" - self.Name = name - self.SnapshotUrl = snapshotUrl - self.StreamUrl = streamUrl - self.FlipH = flipHBool - self.FlipV = flipVBool - self.Rotation = rotationInt - # This is a special flag mostly used for the local plugin webcams to indicate they are no enabled. - self.Enabled = enabled - - - @property - def Name(self): - return self._name - - - @Name.setter - def Name(self, value): - # When the name is set, make sure we convert it to the string style we use internally. - # This ensures that the name can be used and is consistent across the platform. - if value is not None and len(value) > 0: - value = WebcamHelper.MoonrakerToInternalWebcamNameConvert(value) - self._name = value - - - def Validate(self, logger:logging.Logger) -> bool: - if self.Name is None or len(self.Name) == 0: - logger.error(f"Name value in WebcamSettingItem is None or empty. {self.StreamUrl}") - return False - if self.Rotation is None or (self.Rotation != 0 and self.Rotation != 90 and self.Rotation != 180 and self.Rotation != 270): - logger.error(f"Rotation value in WebcamSettingItem is an invalid int. {self.Name} - {self.Rotation}") - return False - if (self.SnapshotUrl is None or len(self.SnapshotUrl) == 0) and (self.StreamUrl is None or len(self.StreamUrl) == 0): - logger.error(f"Snapshot and StreamUrl values in WebcamSettingItem are none or empty {self.Name}") - return False - if self.FlipH is None: - logger.error(f"FlipH value in WebcamSettingItem is None {self.Name}") - return False - self.FlipH = bool(self.FlipH) - if self.FlipV is None: - logger.error(f"FlipV value in WebcamSettingItem is None {self.Name}") - return False - self.FlipV = bool(self.FlipV) - return True - - - # Used to serialize the object to a dict that can be used with json. - # THESE PROPERTY NAMES CAN'T CHANGE, it's used for the API and it's used to serialize to disk. - def Serialize(self, includeUrls:bool = True) -> dict: - d = { - "Name": self.Name, - "FlipH": self.FlipH, - "FlipV": self.FlipV, - "Rotation": self.Rotation, - "Enabled": self.Enabled - } - if includeUrls: - d["SnapshotUrl"] = self.SnapshotUrl - d["StreamUrl"] = self.StreamUrl - return d - - - # Used to convert a dict back into a WebcamSettingItem object. - # Returns None if there's a failure - @staticmethod - def Deserialize(d:dict, logger:logging.Logger): - try: - name = d.get("Name") - snapshotUrl = d.get("SnapshotUrl") - streamUrl = d.get("StreamUrl") - flipH = d.get("FlipH") - flipV = d.get("FlipV") - rotation = d.get("Rotation") - enabled = d.get("Enabled") - if name is None or snapshotUrl is None or streamUrl is None or flipH is None or flipV is None or rotation is None or enabled is None: - raise Exception("Failed to deserialize WebcamSettingItem, missing values.") - i = WebcamSettingItem(str(name), str(snapshotUrl), str(streamUrl), bool(flipH), bool(flipV), int(rotation), bool(enabled)) - if i.Validate(logger) is False: - raise Exception("Failed to validate WebcamSettingItem.") - return i - except Exception as e: - Sentry.Exception("Failed to deserialize WebcamSettingItem", e) - return None - +from ..sentry import Sentry +from ..octohttprequest import OctoHttpRequest +from .webcamsettingitem import WebcamSettingItem +from .quickcam import QuickCamManager # The point of this class is to abstract the logic that needs to be done to reliably get a webcam snapshot and stream from many types of # printer setups. The main entry point is GetSnapshot() which will try a number of ways to get a snapshot from whatever camera system is @@ -114,10 +19,6 @@ class WebcamHelper: # This assumption is also made in the service and website, so it can't change. c_DefaultWebcamIndex = 0 - # We need to cap this so they aren't crazy long. - # However, this COULD mess with teh default camera name logic, since it matches off names. - c_MaxWebcamNameLength = 20 - # A header we apply to all snapshot and webcam streams so the client can get the correct transforms the user has setup. c_OeWebcamTransformHeaderKey = "x-oe-webcam-transform" @@ -128,6 +29,7 @@ class WebcamHelper: @staticmethod def Init(logger:logging.Logger, webcamPlatformHelperInterface, pluginDataFolderPath): WebcamHelper._Instance = WebcamHelper(logger, webcamPlatformHelperInterface, pluginDataFolderPath) + QuickCamManager.Init(logger, webcamPlatformHelperInterface) @staticmethod @@ -146,26 +48,6 @@ def __init__(self, logger:logging.Logger, webcamPlatformHelperInterface, pluginD self._LoadPluginWebcamSettings() - # Returns the snapshot URL from the settings. - # Can be None if there is no snapshot URL set in the settings! - # This URL can be absolute or relative. - def GetSnapshotUrl(self, cameraIndex:int = None): - obj = self._GetWebcamSettingObj(cameraIndex) - if obj is None: - return None - return obj.SnapshotUrl - - - # Returns the mjpeg stream URL from the settings. - # Can be None if there is no URL set in the settings! - # This URL can be absolute or relative. - def GetWebcamStreamUrl(self, cameraIndex:int = None): - obj = self._GetWebcamSettingObj(cameraIndex) - if obj is None: - return None - return obj.StreamUrl - - # Returns if flip H is set in the settings. def GetWebcamFlipH(self, cameraIndex:int = None): obj = self._GetWebcamSettingObj(cameraIndex) @@ -234,15 +116,19 @@ def GetWebcamStream(self, cameraIndex:int = None) -> OctoHttpRequest.Result: def _GetWebcamStreamInternal(self, cameraIndex:int = None) -> OctoHttpRequest.Result: - # Check if the platform helper has an override. - # If so, it CAN BE responsible for all of the snapshot getting logic, but if returns None we should fallback to the default logic. - if hasattr(self.WebcamPlatformHelperInterface, 'GetStream_Override'): - ret = self.WebcamPlatformHelperInterface.GetStream_Override(self._GetWebcamSettingObj(cameraIndex)) - if ret is not None: - return ret + # Get the webcam settings object for this request. + # If there are no webcams, this will return None + webcamSettingsObj = self._GetWebcamSettingObj(cameraIndex) + if webcamSettingsObj is None: + return None + + # First, check if this webcam URL needs to be handled by the QuickCam system. + result = QuickCamManager.Get().TryGetStream(webcamSettingsObj) + if result is not None: + return result # Try to get the URL from the settings. - webcamStreamUrl = self.GetWebcamStreamUrl(cameraIndex) + webcamStreamUrl = webcamSettingsObj.StreamUrl if webcamStreamUrl is not None: # Try to make a standard http call with this stream url # Use use this HTTP call helper system because it might be somewhat tricky to know @@ -268,15 +154,19 @@ def GetSnapshot(self, cameraIndex:int = None) -> OctoHttpRequest.Result: def _GetSnapshotInternal(self, cameraIndex:int = None) -> OctoHttpRequest.Result: - # Check if the platform helper has an override. - # If so, it CAN BE responsible for all of the snapshot getting logic, but if returns None we should fallback to the default logic. - if hasattr(self.WebcamPlatformHelperInterface, 'GetSnapshot_Override'): - ret = self.WebcamPlatformHelperInterface.GetSnapshot_Override(self._GetWebcamSettingObj(cameraIndex)) - if ret is not None: - return ret - - # First, try to get the snapshot using the string defined in settings. - snapshotUrl = self.GetSnapshotUrl(cameraIndex) + # Get the webcam settings object for this request. + # If there are no webcams, this will return None + webcamSettingsObj = self._GetWebcamSettingObj(cameraIndex) + if webcamSettingsObj is None: + return None + + # First, check if this webcam URL needs to be handled by the QuickCam system. + result = QuickCamManager.Get().TryToGetSnapshot(webcamSettingsObj) + if result is not None: + return result + + # Next, try to get the snapshot using the string defined in settings. + snapshotUrl = webcamSettingsObj.SnapshotUrl if snapshotUrl is not None: # Try to make a standard http call with this snapshot url # Use use this HTTP call helper system because it might be somewhat tricky to know @@ -290,7 +180,7 @@ def _GetSnapshotInternal(self, cameraIndex:int = None) -> OctoHttpRequest.Result return octoHttpResult # If getting the snapshot from the snapshot URL fails, try getting a single frame from the mjpeg stream - streamUrl = self.GetWebcamStreamUrl() + streamUrl = webcamSettingsObj.StreamUrl if streamUrl is None: self.Logger.debug("Snapshot helper failed to get a snapshot from the snapshot URL, but we also don't have a stream URL.") return None @@ -444,6 +334,7 @@ def _GetSnapshotFromStream(self, url) -> OctoHttpRequest.Result: # Returns the default webcam setting object or None if there isn't one. # If there isn't a default webcam name, it's assumed to be the first webcam returned in the list command. + # If there are no webcams, this will return None def _GetWebcamSettingObj(self, cameraIndex:int = None): try: # Get the current list of webcam settings. @@ -814,18 +705,3 @@ def _LoadPluginWebcamSettings(self) -> None: self.Logger.info(f"Webcam settings loaded. Default camera name: {self.DefaultCameraName}, Local Webcam Settings Items: {len(self.LocalPluginWebcamSettingsObjects)}") except Exception as e: self.Logger.error("_LoadDefaultCameraName failed "+str(e)) - - - @staticmethod - def MoonrakerToInternalWebcamNameConvert(name:str): - if name is not None and len(name) > 0: - # Enforce max name length. - if len(name) > WebcamHelper.c_MaxWebcamNameLength: - name = name[WebcamHelper.c_MaxWebcamNameLength] - # Ensure the string is only utf8 - name = name.encode('utf-8', 'ignore').decode('utf-8') - # Make the first letter uppercase - name = name[0].upper() + name[1:] - # If there are any / they will break our UI, so remove them. - name = name.replace("/", "") - return name diff --git a/octoeverywhere/Webcam/webcamsettingitem.py b/octoeverywhere/Webcam/webcamsettingitem.py new file mode 100644 index 0000000..76f8ef7 --- /dev/null +++ b/octoeverywhere/Webcam/webcamsettingitem.py @@ -0,0 +1,117 @@ +import logging + +from ..sentry import Sentry + +# +# A platform agnostic definition of a webcam stream. +# +class WebcamSettingItem: + + # We need to cap this so they aren't crazy long. + # However, this COULD mess with teh default camera name logic, since it matches off names. + c_MaxWebcamNameLength = 20 + + # The snapshotUrl and streamUrl can be relative or absolute. + # + # name must exist. + # snapshotUrl OR streamUrl can be None if the values aren't available, but not both. + # flipHBool & flipVBool & rotationInt must exist. + # rotationInt must be 0, 90, 180, or 270 + def __init__(self, name:str = "", snapshotUrl:str = "", streamUrl:str = "", flipHBool:bool = False, flipVBool:bool = False, rotationInt:int = 0, enabled:bool = True): + self._name = "" + self.Name = name + self.SnapshotUrl = snapshotUrl + self.StreamUrl = streamUrl + self.FlipH = flipHBool + self.FlipV = flipVBool + self.Rotation = rotationInt + # This is a special flag mostly used for the local plugin webcams to indicate they are no enabled. + self.Enabled = enabled + + + @property + def Name(self): + return self._name + + + @Name.setter + def Name(self, value): + # When the name is set, make sure we convert it to the string style we use internally. + # This ensures that the name can be used and is consistent across the platform. + if value is not None and len(value) > 0: + value = self._MoonrakerToInternalWebcamNameConvert(value) + self._name = value + + + def Validate(self, logger:logging.Logger) -> bool: + if self.Name is None or len(self.Name) == 0: + logger.error(f"Name value in WebcamSettingItem is None or empty. {self.StreamUrl}") + return False + if self.Rotation is None or (self.Rotation != 0 and self.Rotation != 90 and self.Rotation != 180 and self.Rotation != 270): + logger.error(f"Rotation value in WebcamSettingItem is an invalid int. {self.Name} - {self.Rotation}") + return False + if (self.SnapshotUrl is None or len(self.SnapshotUrl) == 0) and (self.StreamUrl is None or len(self.StreamUrl) == 0): + logger.error(f"Snapshot and StreamUrl values in WebcamSettingItem are none or empty {self.Name}") + return False + if self.FlipH is None: + logger.error(f"FlipH value in WebcamSettingItem is None {self.Name}") + return False + self.FlipH = bool(self.FlipH) + if self.FlipV is None: + logger.error(f"FlipV value in WebcamSettingItem is None {self.Name}") + return False + self.FlipV = bool(self.FlipV) + return True + + + # Used to serialize the object to a dict that can be used with json. + # THESE PROPERTY NAMES CAN'T CHANGE, it's used for the API and it's used to serialize to disk. + def Serialize(self, includeUrls:bool = True) -> dict: + d = { + "Name": self.Name, + "FlipH": self.FlipH, + "FlipV": self.FlipV, + "Rotation": self.Rotation, + "Enabled": self.Enabled + } + if includeUrls: + d["SnapshotUrl"] = self.SnapshotUrl + d["StreamUrl"] = self.StreamUrl + return d + + + # Used to convert a dict back into a WebcamSettingItem object. + # Returns None if there's a failure + @staticmethod + def Deserialize(d:dict, logger:logging.Logger): + try: + name = d.get("Name") + snapshotUrl = d.get("SnapshotUrl") + streamUrl = d.get("StreamUrl") + flipH = d.get("FlipH") + flipV = d.get("FlipV") + rotation = d.get("Rotation") + enabled = d.get("Enabled") + if name is None or snapshotUrl is None or streamUrl is None or flipH is None or flipV is None or rotation is None or enabled is None: + raise Exception("Failed to deserialize WebcamSettingItem, missing values.") + i = WebcamSettingItem(str(name), str(snapshotUrl), str(streamUrl), bool(flipH), bool(flipV), int(rotation), bool(enabled)) + if i.Validate(logger) is False: + raise Exception("Failed to validate WebcamSettingItem.") + return i + except Exception as e: + Sentry.Exception("Failed to deserialize WebcamSettingItem", e) + return None + + + def _MoonrakerToInternalWebcamNameConvert(self, name:str): + if name is not None and len(name) > 0: + # Enforce max name length. + if len(name) > WebcamSettingItem.c_MaxWebcamNameLength: + name = name[WebcamSettingItem.c_MaxWebcamNameLength] + # Ensure the string is only utf8 + name = name.encode('utf-8', 'ignore').decode('utf-8') + # Make the first letter uppercase + name = name[0].upper() + name[1:] + # If there are any / they will break our UI, so remove them. + name = name.replace("/", "") + return name diff --git a/octoeverywhere/Webcam/webcamstreaminstance.py b/octoeverywhere/Webcam/webcamstreaminstance.py new file mode 100644 index 0000000..cf8a738 --- /dev/null +++ b/octoeverywhere/Webcam/webcamstreaminstance.py @@ -0,0 +1,80 @@ +import time +import logging +import threading + +from ..octohttprequest import OctoHttpRequest + +# Stream Instance is a class that is created per web stream to handle streaming QuickCam images into the http stream. +class WebcamStreamInstance: + + # The string doesn't matter what it is, but we define it so it's consistent + c_OeStreamBoundaryString = "oestreamboundary" + + + def __init__(self, logger:logging.Logger, quickCam) -> None: + self.Logger = logger + self.QuickCam = quickCam + self.IsFirstSend = True + self.StreamOpenTimeSec = time.time() + self.ImageReadyEvent = threading.Event() + self.AwaitingImage:bytearray = None + + + # This will attempt to start a stream of the webcam. + # On success, it will return an OctoHttpRequest.Result object with a data callback setup. + # On failure, it will return None. + def StartWebRequest(self) -> OctoHttpRequest.Result: + # First, try to get a snapshot. This will determine if we are able to get a stream or not. + # If we can't start the stream, then we don't return success. + # We will also use this first image to start the stream, to get it going ASAP. + self.AwaitingImage = self.QuickCam.GetCurrentImage() + if self.AwaitingImage is None: + return None + + # Note! We must be sure to call DetachImageStreamCallback to remove this stream callback! + self.QuickCam.AttachImageStreamCallback(self._NewImageCallback) + + # We must set the content type so that the web browser knows what kind of stream to expect. + headers = { + "content-type": f"multipart/x-mixed-replace; boundary={WebcamStreamInstance.c_OeStreamBoundaryString}", + } + + # Return a result object with out callbacks setup for the stream body. + return OctoHttpRequest.Result(200, headers, WebcamStreamInstance.c_OeStreamBoundaryString, False, customBodyStreamCallback=self._CustomBodyStreamRead, customBodyStreamClosedCallback=self._CustomBodyStreamClosed) + + + # Define the callback we will get from QuickCam when there's a new image ready for us to send. + def _NewImageCallback(self, imgBuffer:bytearray): + self.AwaitingImage = imgBuffer + self.ImageReadyEvent.set() + + + # Define a callback for our http body reading system to call when it needs data. + def _CustomBodyStreamRead(self) -> bytearray: + while True: + # See if we can capture an image. There might already be a new image we don't even have to wait for. + capturedImage = self.AwaitingImage + if capturedImage is not None: + # If so, clear the awaiting image and reset the event. + self.AwaitingImage = None + self.ImageReadyEvent.clear() + + # Build the buffer to send + header = f"--{WebcamStreamInstance.c_OeStreamBoundaryString}\r\nContent-Type: image/jpeg\r\nContent-Length: {len(capturedImage)}\r\n\r\n" + imageChunkBuffer = header.encode('utf-8') + capturedImage + b"\r\n" + header.encode('utf-8') + capturedImage + b"\r\n" + + # TODO - I don't know why, but chrome seems to delay the rendering of the image until it gets two? + # This could be something in the pipeline not flushing correctly, or other things. But for now, on the first send we double the image to make it render instantly. + if self.IsFirstSend: + imageChunkBuffer = imageChunkBuffer + imageChunkBuffer + self.IsFirstSend = False + self.Logger.info(f"QuickCam took {time.time()-self.StreamOpenTimeSec} seconds from stream open to first image sent.") + return imageChunkBuffer + # If we didn't get an image, wait on the event for a new one. + self.ImageReadyEvent.wait() + + + # Define a callback for when the http stream is closed. + def _CustomBodyStreamClosed(self) -> None: + # It's important this is called so the stream will be detached! + self.QuickCam.DetachImageStreamCallback(self._NewImageCallback) diff --git a/octoeverywhere/commandhandler.py b/octoeverywhere/commandhandler.py index 993a14c..882c0ec 100644 --- a/octoeverywhere/commandhandler.py +++ b/octoeverywhere/commandhandler.py @@ -3,7 +3,8 @@ from .octostreammsgbuilder import OctoStreamMsgBuilder from .octohttprequest import OctoHttpRequest from .octohttprequest import PathTypes -from .webcamhelper import WebcamHelper, WebcamSettingItem +from .Webcam.webcamhelper import WebcamHelper +from .Webcam.webcamsettingitem import WebcamSettingItem from .sentry import Sentry # diff --git a/octoeverywhere/notificationshandler.py b/octoeverywhere/notificationshandler.py index a88312a..0b92668 100644 --- a/octoeverywhere/notificationshandler.py +++ b/octoeverywhere/notificationshandler.py @@ -13,7 +13,7 @@ from .compat import Compat from .finalsnap import FinalSnap from .repeattimer import RepeatTimer -from .webcamhelper import WebcamHelper +from .Webcam.webcamhelper import WebcamHelper from .printinfo import PrintInfoManager, PrintInfo from .snapshotresizeparams import SnapshotResizeParams diff --git a/octoprint_octoeverywhere/__init__.py b/octoprint_octoeverywhere/__init__.py index df7567e..e412f79 100644 --- a/octoprint_octoeverywhere/__init__.py +++ b/octoprint_octoeverywhere/__init__.py @@ -9,7 +9,7 @@ import requests import octoprint.plugin -from octoeverywhere.webcamhelper import WebcamHelper +from octoeverywhere.Webcam.webcamhelper import WebcamHelper from octoeverywhere.octoeverywhereimpl import OctoEverywhere from octoeverywhere.octohttprequest import OctoHttpRequest from octoeverywhere.notificationshandler import NotificationsHandler diff --git a/octoprint_octoeverywhere/__main__.py b/octoprint_octoeverywhere/__main__.py index e56363a..2d53275 100644 --- a/octoprint_octoeverywhere/__main__.py +++ b/octoprint_octoeverywhere/__main__.py @@ -4,7 +4,7 @@ import random import string -from octoeverywhere.webcamhelper import WebcamHelper +from octoeverywhere.Webcam.webcamhelper import WebcamHelper from octoeverywhere.octoeverywhereimpl import OctoEverywhere from octoeverywhere.octohttprequest import OctoHttpRequest from octoeverywhere.commandhandler import CommandHandler diff --git a/octoprint_octoeverywhere/octoprintwebcamhelper.py b/octoprint_octoeverywhere/octoprintwebcamhelper.py index aa87141..f2bb558 100644 --- a/octoprint_octoeverywhere/octoprintwebcamhelper.py +++ b/octoprint_octoeverywhere/octoprintwebcamhelper.py @@ -2,7 +2,8 @@ import json import time -from octoeverywhere.webcamhelper import WebcamSettingItem, WebcamHelper +from octoeverywhere.Webcam.webcamhelper import WebcamHelper +from octoeverywhere.Webcam.webcamsettingitem import WebcamSettingItem from octoeverywhere.octohttprequest import OctoHttpRequest from octoeverywhere.sentry import Sentry @@ -204,3 +205,14 @@ def GetWebcamConfig(self): # Return the results. return results + + + # !! Interface Function !! + # This function is called to determine if a QuickCam stream should keep running or not. + # The idea is since a QuickCam stream can take longer to start, for example, the Bambu Websocket stream on sends 1FPS, + # we can keep the stream running while the print is running to lower the latency of getting images. + # Most most platforms, this should return true if the print is running or paused, otherwise false. + # Also consider something like Gadget, it takes pictures every 20-40 seconds, so the stream will be started frequently if it's not already running. + def ShouldQuickCamStreamKeepRunning(self) -> bool: + # TODO - this should return true if we are still printing. + return False diff --git a/py_installer/Ffmpeg.py b/py_installer/Ffmpeg.py index 6136eeb..606aaa6 100644 --- a/py_installer/Ffmpeg.py +++ b/py_installer/Ffmpeg.py @@ -2,13 +2,19 @@ from .Util import Util from .Logging import Logger +from .Context import Context, OsTypes # A helper class to make sure ffmpeg is installed. class Ffmpeg: # Tries to install ffmpeg, but this won't fail if the install fails. @staticmethod - def EnsureFfmpeg(): + def TryToInstallFfmpeg(context:Context): + + # We don't even try installing ffmpeg on K1 or SonicPad. + if context.OsType == OsTypes.K1 or context.OsType == OsTypes.SonicPad: + return + # Try to install or upgrade. Logger.Info("Installing ffmpeg, this might take a moment...") startSec = time.time() @@ -20,5 +26,6 @@ def EnsureFfmpeg(): Logger.Info(f"Ffmpeg successfully installed/updated. It took {str(round(time.time()-startSec, 2))} seconds.") return - # Warn, but don't throw or stop the installer. - Logger.Warn(f"Ffmpeg failed to install. It took {str(round(time.time()-startSec, 2))} seconds. Error: {stdError}") + # Tell the user, but this is a best effort, so if it fails we don't care. + # Any user who wants to use RTSP and doesn't have ffmpeg installed can use our help docs to install it. + Logger.Info(f"We didn't install ffmpeg. It took {str(round(time.time()-startSec, 2))} seconds. Output: {stdError}") diff --git a/py_installer/Installer.py b/py_installer/Installer.py index 6fa6025..c4ee1c7 100644 --- a/py_installer/Installer.py +++ b/py_installer/Installer.py @@ -140,9 +140,9 @@ def _RunInternal(self): frontend = Frontend() frontend.DoFrontendSetup(context) - # If this is a bambu setup, make sure ffmpeg is installed since it's required for the X1 webcam. - if context.IsBambuSetup: - Ffmpeg.EnsureFfmpeg() + # We need ffmpeg for the Bambu Connect X1 streaming or any user who wants to use a RTSP camera. + # Installing ffmpeg is best effort and not required for the plugin to work. + Ffmpeg.TryToInstallFfmpeg(context) # Before we start the service, check if the secrets config file already exists and if a printer id already exists. # This will indicate if this is a fresh install or not. diff --git a/setup.py b/setup.py index 5e4a02c..076172a 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this is also parsed by the moonraker module to pull the version, so the string and format must remain the same! -plugin_version = "3.3.0" +plugin_version = "3.3.1" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 5cce7eafd37758f68d36a010b1a32ba1e0a3c3f1 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Wed, 1 May 2024 20:20:47 -0700 Subject: [PATCH 080/328] Minor bug fix --- py_installer/Updater.py | 9 ++------- setup.py | 2 +- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/py_installer/Updater.py b/py_installer/Updater.py index 614e3a2..6cc9165 100644 --- a/py_installer/Updater.py +++ b/py_installer/Updater.py @@ -34,24 +34,19 @@ def DoUpdate(self, context:Context): # Note GetServiceFileFolderPath will return dynamically based on the OsType detected. # Use sorted, so the results are in a nice user presentable order. foundOeServices = [] - hasAnyBambuConnects = False fileAndDirList = sorted(os.listdir(Paths.GetServiceFileFolderPath(context))) for fileOrDirName in fileAndDirList: Logger.Debug(f"Searching for OE services to update, found: {fileOrDirName}") fileOrDirNameLower = fileOrDirName.lower() if Configure.c_ServiceCommonName in fileOrDirNameLower: foundOeServices.append(fileOrDirName) - if "bambu" in fileOrDirNameLower: - hasAnyBambuConnects = True if len(foundOeServices) == 0: Logger.Warn("No local, companion, or Bambu Connect plugins were found on this device.") raise Exception("No local, companion, or Bambu Connect plugins were found on this device.") - # If this is a bambu system, we also want to make sure we install/upgrade ffmpeg - # Since it's required for the X1 camera streaming. - if hasAnyBambuConnects: - Ffmpeg.EnsureFfmpeg() + # On any system, try to install or update ffmpeg. + Ffmpeg.TryToInstallFfmpeg(context) Logger.Info("We found the following plugins to update:") for s in foundOeServices: diff --git a/setup.py b/setup.py index 076172a..b29f291 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this is also parsed by the moonraker module to pull the version, so the string and format must remain the same! -plugin_version = "3.3.1" +plugin_version = "3.3.2" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 580eb6cc8e221b698b172fbb69768a4f2cb2b845 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Wed, 1 May 2024 20:27:46 -0700 Subject: [PATCH 081/328] A minor bug fix for OctoPrint. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index b29f291..4e1b1ce 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this is also parsed by the moonraker module to pull the version, so the string and format must remain the same! -plugin_version = "3.3.2" +plugin_version = "3.3.3" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module @@ -98,7 +98,7 @@ # Any additional python packages you need to install with your plugin that are not contained in .* # For OctoEverywhere, we need to include or common packages shared between hosts, so OctoPrint copies them into the package folder as well. -plugin_additional_packages = [ "octoeverywhere", "octoeverywhere.Proto", "octoeverywhere.WebStream" ] +plugin_additional_packages = [ "octoeverywhere", "octoeverywhere.Proto", "octoeverywhere.WebStream", "octoeverywhere.Webcam" ] # Any python packages within .* you do NOT want to install with your plugin plugin_ignored_packages = [] From 0fdaebdab70c7f225c1b3f5e8893cb294c2e322c Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Thu, 2 May 2024 22:03:52 -0700 Subject: [PATCH 082/328] Fixing a few final X1 bugs for the Bambu Lab release! --- bambu_octoeverywhere/bambuclient.py | 8 ++++++-- linux_host/networksearch.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/bambu_octoeverywhere/bambuclient.py b/bambu_octoeverywhere/bambuclient.py index b327f60..f43d5c2 100644 --- a/bambu_octoeverywhere/bambuclient.py +++ b/bambu_octoeverywhere/bambuclient.py @@ -107,6 +107,10 @@ def _ClientWorker(self): self.Client.tls_insecure_set(True) self.Client.username_pw_set("bblp", self.AccessToken) + # Since we are local, we can do more aggressive reconnect logic. + # The default is min=1 max=120 seconds. + self.Client.reconnect_delay_set(min_delay=1, max_delay=5) + # Setup the callback functions. self.Client.on_connect = self._OnConnect self.Client.on_message = self._OnMessage @@ -120,7 +124,7 @@ def _ClientWorker(self): # Connect to the server # This will throw if it fails, but after that, the loop_forever will handle reconnecting. localBackoffCounter += 1 - self.Client.connect(ipOrHostname, int(self.PortStr), keepalive=60) + self.Client.connect(ipOrHostname, int(self.PortStr), keepalive=5) # Note that self.Client.connect will not throw if there's no MQTT server, but not if auth is wrong. # So if it didn't throw, we know there's a server there, but it might not be the right server @@ -310,7 +314,7 @@ def _Publish(self, msg:dict) -> bool: return False # Try to publish. - state = self.Client.publish(f"device/{self.PrinterSn}/report", json.dumps(msg)) + state = self.Client.publish(f"device/{self.PrinterSn}/request", json.dumps(msg)) # Wait for the message publish to be acked. # This will throw if the publish fails. diff --git a/linux_host/networksearch.py b/linux_host/networksearch.py index 63d7648..87660dd 100644 --- a/linux_host/networksearch.py +++ b/linux_host/networksearch.py @@ -105,7 +105,7 @@ def subscribe(client, userdata:dict, mid, reason_code_list:List[mqtt.ReasonCode] userdata["SnSubSuccess"] = True logger.debug(f"Bambu {ipOrHostname} Sub success.") # Push the message to get the full state, this is needed on teh P1 and A1 - client.publish(f"device/{printerSn}/report", json.dumps( { "pushing": {"sequence_id": "0", "command": "pushall"}})) + client.publish(f"device/{printerSn}/request", json.dumps( { "pushing": {"sequence_id": "0", "command": "pushall"}})) # Check if we are done, this will disconnect if we are. NetworkSearch._BambuConnectionDone(userdata, client) From fdcf10df7c4fd08560af8ac970c4142dfee343fa Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Thu, 2 May 2024 22:15:33 -0700 Subject: [PATCH 083/328] Minor QuickCam fixes. --- octoeverywhere/Webcam/quickcam.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/octoeverywhere/Webcam/quickcam.py b/octoeverywhere/Webcam/quickcam.py index 3a1e66f..9bb3818 100644 --- a/octoeverywhere/Webcam/quickcam.py +++ b/octoeverywhere/Webcam/quickcam.py @@ -481,24 +481,31 @@ def Connect(self, url:str) -> None: # We set the logging level of ffmpeg depending on our logging level # The logs are written to stderr even if they aren't errors, which is nice, so # we can capture them on timeouts. - logLevel = "trace" if self.Logger.isEnabledFor(logging.DEBUG) else "warning" + #logLevel = "trace" if self.Logger.isEnabledFor(logging.DEBUG) else "warning" + # TODO - anything lower than warning has too much tax on ffmpeg, so we don't use it for now. + logLevel = "warning" + + # For FPS, we have found that we can stream and transcode the X1 rtsp stream at a smooth 15 fps on a Pi 4. + # But for other RTSP streams like Wzye bridge cams, it's more intensive and we need to drop to 10 fps. + # If we don't drop the FPS, the stream will fall behind. + fps = 10 + if url.find("bblp:") != -1: + fps = 15 # For auth, if there's a username and password it will already be in the URL in the http:// basic auth style, # So there's nothing else we need to do. # Notes - # We use 15 fps because it's a good trade off of fps and cpu perf hits - # It also decreases the bandwidth needed, which helps on mobile - # We use the default jpeg image quality, for the same reasons above. + # We use the default jpeg image quality, for the same FPS reasons above. # pylint: disable=consider-using-with # We handle this on our own. self.Process = subprocess.Popen(["ffmpeg", "-hide_banner", "-y", "-loglevel", logLevel, - "-rtsp_transport", "udp", + "-rtsp_transport", "0", # Use a value of 0, so both TCP and UDP can be used. "-use_wallclock_as_timestamps", "1", "-i", url, - "-filter:v", "fps=15", + "-filter:v", f"fps={fps}", "-movflags", "+faststart", "-f", "image2pipe", "-" ], From 0fd4bfc1c50c35c9fa32697e95200172e6468ffc Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Fri, 3 May 2024 21:41:54 -0700 Subject: [PATCH 084/328] Fixing a small issue with relative Bambu alternative webcam paths. --- bambu_octoeverywhere/bambuhost.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bambu_octoeverywhere/bambuhost.py b/bambu_octoeverywhere/bambuhost.py index 0c47cb2..6370d09 100644 --- a/bambu_octoeverywhere/bambuhost.py +++ b/bambu_octoeverywhere/bambuhost.py @@ -103,6 +103,12 @@ def RunBlocking(self, configPath, localStorageDir, repoRoot, devConfig_CanBeNone # For bambu, there's no frontend to connect to, so we disable the http relay system. OctoHttpRequest.SetDisableHttpRelay(True) + # But we still want to set the "local OctoPrint port" to 80, because that's the default port it will try for relative URLs. + # Relative URLs for Bambu only come from the alternative webcam streaming system, which the user might be trying to access a webcam stream from this device. + # If they don't specify a IP (or localhost) and port, then we will default all relative URLs to the "local OctoPrint port" value. + OctoHttpRequest.SetLocalHttpProxyPort(80) + OctoHttpRequest.SetLocalOctoPrintPort(80) + OctoHttpRequest.SetLocalHttpProxyIsHttps(False) # Init the ping pong helper. OctoPingPong.Init(self.Logger, localStorageDir, printerId) From 9a8a855ddfea2bb4b252e4460809bb964c2f2ab9 Mon Sep 17 00:00:00 2001 From: Cyril Guislain Date: Mon, 13 May 2024 00:14:40 +0200 Subject: [PATCH 085/328] Update install.sh to fix K1 pip cert issues --- install.sh | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/install.sh b/install.sh index e8ecf21..bcafdce 100755 --- a/install.sh +++ b/install.sh @@ -220,7 +220,7 @@ install_or_update_system_dependencies() # But in general, PY will already be installed, so there's no need to try. # On the K1, the only we thing we ensure is that virtualenv is installed via pip. # We have had users report issues where this install gets stuck, using the no cache dir flag seems to fix it. - pip3 install -q --no-cache-dir virtualenv + pip3 install -q --trusted-host pypi.python.org --trusted-host pypi.org --trusted-host=files.pythonhosted.org --no-cache-dir virtualenv elif [[ $IS_SONIC_PAD_OS -eq 1 ]] then # The sonic pad always has opkg installed, so we can make sure these packages are installed. @@ -265,11 +265,21 @@ install_or_update_python_env() # Update pip if needed - we added a note because this takes a while on the sonic pad. log_info "Updating PIP if needed... (this can take a few seconds or so)" - "${OE_ENV}"/bin/python -m pip install --upgrade pip + if [[ $IS_K1_OS -eq 1 ]] + then + "${OE_ENV}"/bin/python -m pip install --trusted-host pypi.python.org --trusted-host pypi.org --trusted-host=files.pythonhosted.org --no-cache-dir --upgrade pip + else + "${OE_ENV}"/bin/python -m pip install --upgrade pip + fi # Finally, ensure our plugin requirements are installed and updated. log_info "Installing or updating required python libs..." - "${OE_ENV}"/bin/pip3 install --require-virtualenv --no-cache-dir -q -r "${OE_REPO_DIR}"/requirements.txt + if [[ $IS_K1_OS -eq 1 ]] + then + "${OE_ENV}"/bin/pip3 install --trusted-host pypi.python.org --trusted-host pypi.org --trusted-host=files.pythonhosted.org --require-virtualenv --no-cache-dir -q -r "${OE_REPO_DIR}"/requirements.txt + else + "${OE_ENV}"/bin/pip3 install --require-virtualenv --no-cache-dir -q -r "${OE_REPO_DIR}"/requirements.txt + fi log_info "Python libs installed." } From 40d5ba24dc46b5b045bb6d369e0b1b1b5d2aa6d1 Mon Sep 17 00:00:00 2001 From: Cyril Guislain Date: Wed, 15 May 2024 05:20:23 +0200 Subject: [PATCH 086/328] More Creality K1 improvements * venv improvements for Creality K1 This PR improve venv with latest python version to avoid future dependencies installation issue on Creality K1. * Update install.sh --- install.sh | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/install.sh b/install.sh index bcafdce..b18870a 100755 --- a/install.sh +++ b/install.sh @@ -88,7 +88,7 @@ PKGLIST="python3 python3-pip virtualenv python3-venv curl" # We don't override the default name, since that's used by the Moonraker installer # Note that we DON'T want to use the same name as above (not even in this comment) because some parsers might find it. # Note we exclude virtualenv python3-venv curl because they can't be installed on the sonic pad via the package manager. -SONIC_PAD_DEP_LIST="python3 python3-pip" +CREALITY_DEP_LIST="python3 python3-pip" # @@ -198,7 +198,12 @@ ensure_py_venv() then # The K1 requires we setup the virtualenv like this. # --system-site-packages is important for the K1, since it doesn't have much disk space. - python3 /usr/lib/python3.8/site-packages/virtualenv.py -p /usr/bin/python3 --system-site-packages "${OE_ENV}" + if [[ -f /opt/bin/python3 ]] + then + virtualenv -p /opt/bin/python3 --system-site-packages "${OE_ENV}" + else + python3 /usr/lib/python3.8/site-packages/virtualenv.py -p /usr/bin/python3 --system-site-packages "${OE_ENV}" + fi else # Everything else can use this more modern style command. # We don't want to use --system-site-packages, so we don't consume whatever packages are on the system. @@ -220,12 +225,16 @@ install_or_update_system_dependencies() # But in general, PY will already be installed, so there's no need to try. # On the K1, the only we thing we ensure is that virtualenv is installed via pip. # We have had users report issues where this install gets stuck, using the no cache dir flag seems to fix it. + if [[ -f /opt/bin/opkg ]] + then + opkg install ${CREALITY_DEP_LIST} + fi pip3 install -q --trusted-host pypi.python.org --trusted-host pypi.org --trusted-host=files.pythonhosted.org --no-cache-dir virtualenv elif [[ $IS_SONIC_PAD_OS -eq 1 ]] then # The sonic pad always has opkg installed, so we can make sure these packages are installed. # We have had users report issues where this install gets stuck, using the no cache dir flag seems to fix it. - opkg install ${SONIC_PAD_DEP_LIST} + opkg install ${CREALITY_DEP_LIST} pip3 install -q --no-cache-dir virtualenv else # It seems a lot of printer control systems don't have the date and time set correctly, and then the fail @@ -421,4 +430,4 @@ if [ $retVal -ne 0 ]; then fi # Note the rest of the user flow (and terminal info) is done by the PY script, so we don't need to report anything else. -exit $retVal \ No newline at end of file +exit $retVal From b7d0841288f0774b9a6f02a79a57f057c68b0bc9 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Tue, 14 May 2024 21:09:56 -0700 Subject: [PATCH 087/328] Adding a few more minor K1 install fixes --- install.sh | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/install.sh b/install.sh index b18870a..c693935 100755 --- a/install.sh +++ b/install.sh @@ -198,6 +198,9 @@ ensure_py_venv() then # The K1 requires we setup the virtualenv like this. # --system-site-packages is important for the K1, since it doesn't have much disk space. + # Ideally we use /opt/bin/python3, since that version of python will be updated over time. + # It installs with the opkg command, if opkg is there. + # If not, we will use the version of python built into the system for the existing Creality stuff. if [[ -f /opt/bin/python3 ]] then virtualenv -p /opt/bin/python3 --system-site-packages "${OE_ENV}" @@ -222,19 +225,23 @@ install_or_update_system_dependencies() then # The K1 by default doesn't have any package manager. In some cases # the user might install opkg via the 3rd party moonraker installer script. - # But in general, PY will already be installed, so there's no need to try. - # On the K1, the only we thing we ensure is that virtualenv is installed via pip. - # We have had users report issues where this install gets stuck, using the no cache dir flag seems to fix it. + # But in general, PY will already be installed. + # We will try to update python from the package manager if possible, otherwise, we will ignore it. if [[ -f /opt/bin/opkg ]] then - opkg install ${CREALITY_DEP_LIST} + opkg update || true + opkg install ${CREALITY_DEP_LIST} || true fi + # On the K1, the only we thing we ensure is that virtualenv is installed via pip. + # We have had users report issues where this install gets stuck, using the no cache dir flag seems to fix it. + # 5/14/24 - The trusted hosts had to be added to fix a cert issue with pypi we aren't sure why it started happening all of the sudden. pip3 install -q --trusted-host pypi.python.org --trusted-host pypi.org --trusted-host=files.pythonhosted.org --no-cache-dir virtualenv elif [[ $IS_SONIC_PAD_OS -eq 1 ]] then # The sonic pad always has opkg installed, so we can make sure these packages are installed. # We have had users report issues where this install gets stuck, using the no cache dir flag seems to fix it. - opkg install ${CREALITY_DEP_LIST} + opkg update || true + opkg install ${CREALITY_DEP_LIST} || true pip3 install -q --no-cache-dir virtualenv else # It seems a lot of printer control systems don't have the date and time set correctly, and then the fail From 2d61792fe3bae28dde895917be3810c4068cb9b4 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Tue, 14 May 2024 21:11:40 -0700 Subject: [PATCH 088/328] Lint fix --- octoeverywhere/octosessionimpl.py | 1 - 1 file changed, 1 deletion(-) diff --git a/octoeverywhere/octosessionimpl.py b/octoeverywhere/octosessionimpl.py index 4fbf22c..a799149 100644 --- a/octoeverywhere/octosessionimpl.py +++ b/octoeverywhere/octosessionimpl.py @@ -253,7 +253,6 @@ def StartHandshake(self, summonMethod): except Exception as e: Sentry.Exception("Failed to send handshake syn.", e) self.OnSessionError(0) - return # This is the main receive function for all messages coming from the server. From 554f6e0669cc18e39ee6ae3d1e8aa52e90032956 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Tue, 14 May 2024 21:26:22 -0700 Subject: [PATCH 089/328] Fixng a bug where the 1st and 3rd layer complete commands fire too soon on the Bambu Lab printers. Also adding more debugging. --- bambu_octoeverywhere/bambumodels.py | 7 ++++++- bambu_octoeverywhere/bambustatetranslater.py | 7 ++++++- moonraker_octoeverywhere/moonrakerwebcamhelper.py | 1 + octoeverywhere/Webcam/quickcam.py | 4 +--- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/bambu_octoeverywhere/bambumodels.py b/bambu_octoeverywhere/bambumodels.py index d3abdae..f0aa514 100644 --- a/bambu_octoeverywhere/bambumodels.py +++ b/bambu_octoeverywhere/bambumodels.py @@ -73,7 +73,7 @@ def GetContinuousTimeRemainingSec(self) -> int: if self.mc_remaining_time is None or self.LastTimeRemainingWallClock is None: return None # The slicer holds a constant time while in preparing, so we don't want to fake our countdown either. - if self.gcode_state == "SLICING" or self.gcode_state == "PREPARE": + if self.IsPrepareOrSlicing(): # Reset the last wall clock time to now, so when we transition to running, we don't snap to a strange offset. self.LastTimeRemainingWallClock = time.time() return int(self.mc_remaining_time * 60) @@ -102,6 +102,11 @@ def IsPrintingState(state:str, includePausedAsPrinting:bool) -> bool: return state == "RUNNING" or BambuState.IsPrepareOrSlicingState(state) + # We use this common method to keep all of the logic common in the plugin + def IsPrepareOrSlicing(self) -> bool: + return BambuState.IsPrepareOrSlicingState(self.gcode_state) + + # We use this common method to keep all of the logic common in the plugin @staticmethod def IsPrepareOrSlicingState(state:str) -> bool: diff --git a/bambu_octoeverywhere/bambustatetranslater.py b/bambu_octoeverywhere/bambustatetranslater.py index 0261fda..029ffe9 100644 --- a/bambu_octoeverywhere/bambustatetranslater.py +++ b/bambu_octoeverywhere/bambustatetranslater.py @@ -196,7 +196,12 @@ def GetCurrentZOffset(self): def GetCurrentLayerInfo(self): state = BambuClient.Get().GetState() if state is None: - return (None, None) + # If we dont have a state yet, return 0,0, which means we can get layer info but we don't know yet. + return (0, 0) + if state.IsPrepareOrSlicing(): + # The printer doesn't clear these values when a new print is starting and it's in a prepare or slicing state. + # So if we are in that state, return 0,0, to represent we don't know the layer info yet. + return (0, 0) # We can get accurate and 100% correct layers from Bambu, awesome! currentLayer = None totalLayers = None diff --git a/moonraker_octoeverywhere/moonrakerwebcamhelper.py b/moonraker_octoeverywhere/moonrakerwebcamhelper.py index 5e688a2..98b15b5 100644 --- a/moonraker_octoeverywhere/moonrakerwebcamhelper.py +++ b/moonraker_octoeverywhere/moonrakerwebcamhelper.py @@ -219,6 +219,7 @@ def _DoAutoSettingsUpdate(self): if result.HasError(): if "Namespace 'webcams' not found".lower() in result.ErrorStr.lower(): # This happens if there are no webcams configured at all. + self.Logger.debug("server.database.get_item returned no webcam namespace found.") self._ResetValuesToDefaults() return self.Logger.warn("Moonraker webcam helper failed to query for webcams. "+result.GetLoggingErrorStr()) diff --git a/octoeverywhere/Webcam/quickcam.py b/octoeverywhere/Webcam/quickcam.py index 9bb3818..5df057a 100644 --- a/octoeverywhere/Webcam/quickcam.py +++ b/octoeverywhere/Webcam/quickcam.py @@ -481,9 +481,7 @@ def Connect(self, url:str) -> None: # We set the logging level of ffmpeg depending on our logging level # The logs are written to stderr even if they aren't errors, which is nice, so # we can capture them on timeouts. - #logLevel = "trace" if self.Logger.isEnabledFor(logging.DEBUG) else "warning" - # TODO - anything lower than warning has too much tax on ffmpeg, so we don't use it for now. - logLevel = "warning" + logLevel = "trace" if self.Logger.isEnabledFor(logging.DEBUG) else "warning" # For FPS, we have found that we can stream and transcode the X1 rtsp stream at a smooth 15 fps on a Pi 4. # But for other RTSP streams like Wzye bridge cams, it's more intensive and we need to drop to 10 fps. From ef50344709285eb79046db1bb9bdd28637df4719 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Tue, 14 May 2024 21:28:47 -0700 Subject: [PATCH 090/328] Version bump --- .vscode/settings.json | 1 + bambu_octoeverywhere/bambumodels.py | 7 +------ setup.py | 4 ++-- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 2bc68e9..4ab50ed 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -233,6 +233,7 @@ "websocketimpl", "websockets", "webstream", + "Wzye", "zchange", "zhop", "zhops", diff --git a/bambu_octoeverywhere/bambumodels.py b/bambu_octoeverywhere/bambumodels.py index f0aa514..db3a52f 100644 --- a/bambu_octoeverywhere/bambumodels.py +++ b/bambu_octoeverywhere/bambumodels.py @@ -86,11 +86,6 @@ def IsPrinting(self, includePausedAsPrinting:bool) -> bool: return BambuState.IsPrintingState(self.gcode_state, includePausedAsPrinting) - # Since there's a lot to consider to figure out if a print is running, this one function acts as common logic across the plugin. - def IsPrepareOrSlicing(self) -> bool: - return BambuState.IsPrepareOrSlicingState(self.gcode_state) - - # We use this common method since "is this a printing state?" is complicated and we can to keep all of the logic common in the plugin @staticmethod def IsPrintingState(state:str, includePausedAsPrinting:bool) -> bool: @@ -163,7 +158,7 @@ def GetPrinterError(self) -> BambuPrintErrors: # There's a full list of errors here, we only care about some of them # https://e.bambulab.com/query.php?lang=en # We format the error into a hex the same way the are on the page, to make it easier. - # NOTE SOME ERRORS HAVE MULTPLE VALUES, SO GET THEM ALL! + # NOTE SOME ERRORS HAVE MULTIPLE VALUES, SO GET THEM ALL! # They have different values for the different AMS slots h = hex(self.print_error)[2:].rjust(8, '0') errorMap = { diff --git a/setup.py b/setup.py index 4e1b1ce..beb7b68 100644 --- a/setup.py +++ b/setup.py @@ -29,8 +29,8 @@ plugin_name = "OctoEverywhere" # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module -# Note that this is also parsed by the moonraker module to pull the version, so the string and format must remain the same! -plugin_version = "3.3.3" +# Note that this single version string is used by all of the plugins in OctoEverywhere! +plugin_version = "3.3.4" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From fb2d46ddcd3a0091730d5c61f5fbdeab972ec0d1 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sun, 19 May 2024 10:53:05 -0700 Subject: [PATCH 091/328] Adding an official docker image for Bambu Connect! --- .github/workflows/docker-publish.yml | 60 +++++++++ .github/workflows/pylint.yml | 3 +- .vscode/settings.json | 2 + Dockerfile | 37 +++++ bambu_octoeverywhere/bambuclient.py | 11 +- docker-compose.yml | 28 ++++ docker-readme.md | 62 +++++++++ docker_octoeverywhere/__init__.py | 1 + docker_octoeverywhere/__main__.py | 195 +++++++++++++++++++++++++++ install.sh | 10 +- 10 files changed, 400 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/docker-publish.yml create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100644 docker-readme.md create mode 100644 docker_octoeverywhere/__init__.py create mode 100644 docker_octoeverywhere/__main__.py diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 0000000..6fd5da9 --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,60 @@ +name: Publish Docker image + +# Only make and deploy new docker images on tagged releases. +on: + release: + types: [published] + +jobs: + push_to_registry: + name: Push Docker image to Docker Hub + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + # This is needed for the attestation step + id-token: write + attestations: write + steps: + - name: Check out the repo + uses: actions/checkout@v4 + + # Required for docker multi arch building. + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + # Required for docker multi arch building. + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: octoeverywhere/octoeverywhere + + - name: Build and push Docker image + id: push + uses: docker/build-push-action@v5 + with: + context: . + # These are the platform our ubuntu image base supports + platforms: linux/amd64,linux/arm/v7,linux/arm64 + file: ./Dockerfile + push: true + tags: octoeverywhere/octoeverywhere:latest + labels: ${{ steps.meta.outputs.labels }} + + # This isn't working, so it's disabled for now. + # - name: Generate artifact attestation + # uses: actions/attest-build-provenance@v1 + # with: + # subject-name: octoeverywhere/octoeverywhere + # subject-digest: ${{ steps.push.outputs.digest }} + # push-to-registry: true diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 85520d0..cf93885 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -29,4 +29,5 @@ jobs: pylint ./moonraker_octoeverywhere/ pylint ./bambu_octoeverywhere/ pylint ./linux_host/ - pylint ./py_installer/ \ No newline at end of file + pylint ./py_installer/ + pylint ./docker_octoeverywhere/ \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index 4ab50ed..f020be3 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -23,6 +23,7 @@ "boundarydonotcross", "brotli", "Buildroot", + "buildx", "certifi", "checkin", "classicwebcam", @@ -61,6 +62,7 @@ "Frontends", "frontendsetup", "fsensor", + "fstring", "gcode", "geteuid", "getpwnam", diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f5f99dd --- /dev/null +++ b/Dockerfile @@ -0,0 +1,37 @@ +# Start with the lastest ubuntu, for a solid base, +# since we need some advance binaries for things like pillow and ffmpeg. +FROM ubuntu:latest + +# We will base ourselves in root, becuase why not. +WORKDIR /root + +# Define some user vars we will use for the image. +# These are read in the docker_octoeverywhere module, so they must not change! +ENV USER=root +ENV REPO_DIR=/root/octoeverywhere +ENV VENV_DIR=/root/octoeverywhere-env +# This is a special dir that the user MUST mount to the host, so that the data is persisted. +# If this is not mounted, the printer will need to be re-linked everytime the container is remade. +ENV DATA_DIR=/data/ + +# Install the required packages. +# Any packages here should be mirrored in the install script - and any optaionl pillow packages done inline. +RUN apt update +RUN apt install -y curl ffmpeg jq python3 python3-pip python3-venv virtualenv libjpeg-dev zlib1g-dev python3-pil python3-pillow + +# +# We decided to not run the installer, since the point of the installer is to setup the env, build the launch args, and setup the serivce. +# Instead, we will manually run the smaller subset of commands that are requred to get the env setup in docker. +# Note that if this ever becomes too much of a hassle, we might want to revert back to using the installer, and supporting a headless install. +# +RUN virtualenv -p /usr/bin/python3 ${VENV_DIR} +RUN ${VENV_DIR}/bin/python -m pip install --upgrade pip + +# Copy the entire repo into the image, do this as late as possible to avoid rebuilding the image everytime the repo changes. +COPY ./ ${REPO_DIR}/ +RUN ${VENV_DIR}/bin/pip3 install --require-virtualenv --no-cache-dir -q -r ${REPO_DIR}/requirements.txt + +# For docker, we use our docker_octoeverywhere host to handle the runtime setup and launch of the serivce. +WORKDIR ${REPO_DIR} +# Use the full path to the venv, we msut use this [] notation for our ctlc handler to work in the contianer +ENTRYPOINT ["/root/octoeverywhere-env/bin/python", "-m", "docker_octoeverywhere"] \ No newline at end of file diff --git a/bambu_octoeverywhere/bambuclient.py b/bambu_octoeverywhere/bambuclient.py index f43d5c2..bed8d0d 100644 --- a/bambu_octoeverywhere/bambuclient.py +++ b/bambu_octoeverywhere/bambuclient.py @@ -48,11 +48,10 @@ def __init__(self, logger:logging.Logger, config:Config, stateTranslator) -> Non # Get the required args. self.Config = config - ipOrHostname = config.GetStr(Config.SectionCompanion, Config.CompanionKeyIpOrHostname, None) self.PortStr = config.GetStr(Config.SectionCompanion, Config.CompanionKeyPort, None) self.AccessToken = config.GetStr(Config.SectionBambu, Config.BambuAccessToken, None) self.PrinterSn = config.GetStr(Config.SectionBambu, Config.BambuPrinterSn, None) - if ipOrHostname is None or self.PortStr is None or self.AccessToken is None or self.PrinterSn is None: + if self.PortStr is None or self.AccessToken is None or self.PrinterSn is None: raise Exception("Missing required args from the config") # We use this var to keep track of consecutively failed connections @@ -147,7 +146,7 @@ def _ClientWorker(self): self.Logger.error(f"Failed to connect to the Bambu printer {ipOrHostname}:{self.PortStr} due to a timeout, we will retry in a bit. "+str(e)) else: # Random other errors. - Sentry.Exception("Failed to connect to the Bambu printer {ipOrHostname}:{self.PortStr}. We will retry in a bit.", e) + Sentry.Exception(f"Failed to connect to the Bambu printer {ipOrHostname}:{self.PortStr}. We will retry in a bit.", e) # Sleep for a bit between tries. # The main consideration here is to not log too much when the printer is off. But we do still want to connect quickly, when it's back on. @@ -333,13 +332,15 @@ def _GetIpOrHostnameToTry(self) -> str: self.ConsecutivelyFailedConnectionAttempts = 0 # On the first few attempts, use the expected IP. - # The first attempt will always be attempt 1, since it's reset to 0 and incremented before connecting + # The first attempt will always be attempt 1, since it's reset to 0 and incremented before connecting. + # The IP can be empty, like if the docker container is used, in which case we should always search for the printer. configIpOrHostname = self.Config.GetStr(Config.SectionCompanion, Config.CompanionKeyIpOrHostname, None) - if self.ConsecutivelyFailedConnectionAttempts < 3: + if configIpOrHostname is not None and len(configIpOrHostname) > 0 and self.ConsecutivelyFailedConnectionAttempts < 3: return configIpOrHostname # If we fail too many times, try to scan for the printer on the local subnet, the IP could have changed. # Since we 100% identify the printer by the access token and printer SN, we can try to scan for it. + self.Logger.info(f"Searching for your Bambu Lab printer {self.PrinterSn}") ips = NetworkSearch.ScanForInstances_Bambu(self.Logger, self.AccessToken, self.PrinterSn) # If we get an IP back, it is the printer. diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..653737e --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,28 @@ +version: '2' +services: + octoeverywhere-bambu-connect: + image: octoeverywhere/octoeverywhere:latest + environment: + # https://octoeverywhere.com/s/access-code + - ACCESS_CODE=XXXXXXXX + # https://octoeverywhere.com/s/bambu-sn + - SERIAL_NUMBER=XXXXXXXXXXXXXXX + # Find using the printer's display + - PRINTER_IP=192.168.1.1 + volumes: + # Specify a path mapping for the required persistent storage + - /some/path/on/your/computer:/data + + # Add as many printers as you want! + # octoeverywhere-bambu-connect-2: + # image: octoeverywhere/octoeverywhere:latest + # environment: + # # https://octoeverywhere.com/s/access-code + # - ACCESS_CODE=XXXXXXXX + # # https://octoeverywhere.com/s/bambu-sn + # - SERIAL_NUMBER=XXXXXXXXXXXXXXX + # # Find using the printer's display + # - PRINTER_IP=192.168.1.2 + # volumes: + # # Specify a path mapping for the required persistent storage + # - /some/path/on/your/computer/printer2:/data \ No newline at end of file diff --git a/docker-readme.md b/docker-readme.md new file mode 100644 index 0000000..a8baa12 --- /dev/null +++ b/docker-readme.md @@ -0,0 +1,62 @@ +# Bambu Connect Docker Support + +OctoEverywhere's docker image only works for [Bambu Connect](https://octoeverywhere.com/bambu?source=github_docker_readme) for Bambu Lab 3D printers. If you are using OctoPrint or Klipper, [follow our getting started guide](https://octoeverywhere.com/getstarted?source=github_docker_readme) to install the OctoEverywhere plugin. + +Official Docker Image: https://hub.docker.com/r/octoeverywhere/octoeverywhere + +## Required Setup Environment Vars + +To use the Bambu Connect plugin, you need to get the following information. + +- Your printer's Access Code - https://octoeverywhere.com/s/access-code +- Your printer's Serial Number - https://octoeverywhere.com/s/bambu-sn +- Your printer's IP Address - (use the printer's display) + +These three values must be set at environment vars when you first run the container. Once the container is run, you don't need to include them again, unless you want to update the values. + +- ACCESS_CODE=(code) +- SERIAL_NUMBER=(serial number) +- PRINTER_IP=(ip address) + +## Required Persistent Storage + +You must map the `/data` folder in your docker container to a directory on your computer so the plugin can write data that will remain between runs. Failure to do this will require relinking the plugin when the container is destroyed or updated. + +## Linking Your Bambu Connect Plugin + +Once the docker container is running, you need to look at the logs to find the linking URL. + +Docker Compose: +`docker-compose logs | grep https://octoeverywhere.com/getstarted` + +Docker: +`docker logs bambu-connect | grep https://octoeverywhere.com/getstarted` + +# Running The Docker Image + +## Using Docker Compose + +Using docker compose is the easiest way to run OctoEverywhere's Bambu Connect using docker. + +- Install [Docker and Docker Compose](https://docs.docker.com/compose/install/linux/) +- Clone this repo +- Edit the `./docker-compose.yml` file to enter your environment vars +- Run `docker-compose up -d` +- Follow the "Linking Your Bambu Connect Plugin" to link the plugin to your account. + +## Using Docker + +Docker compose is a fancy wrapper to run docker containers. You can also run docker containers manually. + +Use a command like this example, but update the required vars. + +`docker pull octoeverywhere/octoeverywhere` +`docker run --name bambu-connect -e ACCESS_CODE= -e SERIAL_NUMBER= -e PRINTER_IP= -v /your/local/path:/data -d octoeverywhere/octoeverywhere` + +Follow the "Linking Your Bambu Connect Plugin" to link the plugin to your account. + +## Building The Image Locally + +You can build the docker image locally if you prefer, use the following command. + +`docker build -t octoeverywhere .` \ No newline at end of file diff --git a/docker_octoeverywhere/__init__.py b/docker_octoeverywhere/__init__.py new file mode 100644 index 0000000..564091d --- /dev/null +++ b/docker_octoeverywhere/__init__.py @@ -0,0 +1 @@ +# Need to make this a module diff --git a/docker_octoeverywhere/__main__.py b/docker_octoeverywhere/__main__.py new file mode 100644 index 0000000..863123a --- /dev/null +++ b/docker_octoeverywhere/__main__.py @@ -0,0 +1,195 @@ +import os +import sys +import json +import time +import signal +import base64 +import logging +import traceback +import subprocess + +# +# This docker host is the entry point for the docker container. +# Unlike the other host, this host doesn't run the service, it invokes the bambu or companion host. +# + +from linux_host.startup import Startup +from linux_host.config import Config + +# pylint: disable=logging-fstring-interpolation + +if __name__ == '__main__': + + # Setup a basic logger + logger = logging.getLogger() + logger.setLevel(logging.DEBUG) + formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') + std = logging.StreamHandler(sys.stdout) + std.setFormatter(formatter) + logger.addHandler(std) + + logger.info("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + logger.info("Starting Docker OctoEverywhere Bootstrap") + logger.info("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + + # This is a helper class, to keep the startup logic common. + s = Startup() + + # + # Helper functions + # + def LogException(msg:str, e:Exception) -> None: + tb = traceback.format_exc() + exceptionClassType = "unknown_type" + if e is not None: + exceptionClassType = e.__class__.__name__ + logger.error(f"{msg}; {str(exceptionClassType)} Exception: {str(e)}; {str(tb)}") + + def EnsureIsPath(path: str) -> str: + logger.info(f"Ensuring path exists: {path}") + if path is None or not os.path.exists(path): + raise Exception(f"Path does not exist: {path}") + return path + + def CreateDirIfNotExists(path: str) -> None: + if not os.path.exists(path): + os.makedirs(path) + + try: + # First, read the required env vars that are set in the dockerfile. + logger.info(f"Env Vars: {os.environ}") + virtualEnvPath = EnsureIsPath(os.environ.get("VENV_DIR", None)) + repoRootPath = EnsureIsPath(os.environ.get("REPO_DIR", None)) + dataPath = EnsureIsPath(os.environ.get("DATA_DIR", None)) + + # For Bambu Connect, the config sits int the data dir. + configPath = dataPath + + # Create the config object, which will read an existing config or make a new one. + # If this is the first run, there will be no config file, so we need to create one. + logger.info(f"Init config object: {configPath}") + config = Config(configPath) + + # If there is a arg passed, always update or set it. + # This allows users to update the values after the image has ran the first time. + accessCode = os.environ.get("ACCESS_CODE", None) + if accessCode is not None: + logger.info(f"Setting Access Code: {accessCode}") + config.SetStr(Config.SectionBambu, Config.BambuAccessToken, accessCode) + # Ensure something is set now. + if config.GetStr(Config.SectionBambu, Config.BambuAccessToken, None) is None: + logger.error("") + logger.error("") + logger.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + logger.error(" You must pass the printer's Access Code as an env var.") + logger.error(" Use `docker run -e ACCESS_CODE=` or add it to your docker-compose file.") + logger.error("") + logger.error(" To find your Access Code -> https://octoeverywhere.com/s/access-code") + logger.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + logger.error("") + logger.error("") + # Sleep some, so we don't restart super fast and then exit. + time.sleep(5.0) + sys.exit(1) + + printerSn = os.environ.get("SERIAL_NUMBER", None) + if printerSn is not None: + logger.info(f"Setting Serial Number: {printerSn}") + config.SetStr(Config.SectionBambu, Config.BambuPrinterSn, printerSn) + # Ensure something is set now. + if config.GetStr(Config.SectionBambu, Config.BambuPrinterSn, None) is None: + logger.error("") + logger.error("") + logger.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + logger.error(" You must pass the printer's Serial Number as an env var.") + logger.error("Use `docker run -e SERIAL_NUMBER=` or add it to your docker-compose file.") + logger.error("") + logger.error(" To find your Serial Number -> https://octoeverywhere.com/s/bambu-sn") + logger.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + logger.error("") + logger.error("") + # Sleep some, so we don't restart super fast and then exit. + time.sleep(5.0) + sys.exit(1) + + # + # If we got here, the access token and serial number are set or were already set. + # We should be able to launch! + # + + # TEMP - Until we fix the issue where the plugin doesn't know the local LAN network address range, we need the + # user to pass the printer's IP to the plugin, since the auto scanning doesn't work. + # When this is fixed, we no longer need it to be passed. + printerId = os.environ.get("PRINTER_IP", None) + if printerId is not None: + logger.info(f"Setting Printer IP: {printerId}") + config.SetStr(Config.SectionCompanion, Config.CompanionKeyIpOrHostname, printerId) + # Ensure something is set now. + if config.GetStr(Config.SectionCompanion, Config.CompanionKeyIpOrHostname, None) is None: + logger.error("") + logger.error("") + logger.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + logger.error(" You must pass the printer's IP Address as an env var.") + logger.error(" Use `docker run -e PRINTER_IP=` or add it to your docker-compose file.") + logger.error("") + logger.error(" To find your Ip Address, use the display on your printer.") + logger.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + logger.error("") + logger.error("") + # Sleep some, so we don't restart super fast and then exit. + time.sleep(5.0) + sys.exit(1) + + # The port is always the same, so we just set the known Bambu Lab printer port. + if config.GetStr(Config.SectionCompanion, Config.CompanionKeyPort, None) is None: + config.SetStr(Config.SectionCompanion, Config.CompanionKeyPort, "8883") + + # We don't set the IP address of the printer. The Bambu Connect plugin will automatically find the printer + # on the local network using the Access Token and SN. By not setting the value, it will force it to search first. + + # Create the rest of the required dirs based in the data dir, since it's persistent. + localStoragePath = os.path.join(dataPath, "octoeverywhere-store") + CreateDirIfNotExists(localStoragePath) + logDirPath = os.path.join(dataPath, "logs") + CreateDirIfNotExists(logDirPath) + + # Build the launch string + launchConfig = { + "ServiceName" : "octoeverywhere", # Since there's only once service, use the default name. + "CompanionInstanceIdStr" : "1", # Since there's only once service, use the default service id. + "VirtualEnvPath" : virtualEnvPath, + "RepoRootFolder" : repoRootPath, + "LocalFileStoragePath" : localStoragePath, + "LogFolder" : logDirPath, + "ConfigFolder" : configPath, + } + + # Convert the launch string into what's expected. + launchConfigStr = json.dumps(launchConfig) + logger.info(f"Launch config: {launchConfigStr}") + base64EncodedLaunchConfig = base64.urlsafe_b64encode(bytes(launchConfigStr, "utf-8")).decode("utf-8") + + # Setup a ctl-c handler, so the docker container can be closed easily. + def signal_handler(sig, frame): + logger.info("OctoEverywhere Bambu Connect docker container stop requested") + sys.exit(0) + signal.signal(signal.SIGINT, signal_handler) + + # Instead of running the plugin in our process, we decided to launch a different process so it's clean and runs + # just like the plugin normally runs. + pythonPath = os.path.join(virtualEnvPath, os.path.join("bin", "python3")) + logger.info(f"Launch PY path: {pythonPath}") + result:subprocess.CompletedProcess = subprocess.run([pythonPath, "-m", "bambu_octoeverywhere", base64EncodedLaunchConfig], check=False) + + # Normally the process shouldn't exit unless it hits a bad error. + if result.returncode == 0: + logger.info(f"Bambu Connect plugin exited. Result: {result.returncode}") + else: + logger.error(f"Bambu Connect plugin exited with an error. Result: {result.returncode}") + + except Exception as e: + LogException("Exception while bootstrapping up OctoEverywhere Bambu Connect.", e) + + # Sleep for a bit, so if we are restarted we don't do it instantly. + time.sleep(3) + sys.exit(1) diff --git a/install.sh b/install.sh index c693935..0deb56a 100755 --- a/install.sh +++ b/install.sh @@ -83,13 +83,14 @@ OE_ENV="${HOME}/octoeverywhere-env" # The virtualenv is for our virtual package env we create # The curl requirement is for some things in this bootstrap script. # python3-venv is required for teh virtualenv command to fully work. +# This must stay in sync with the dockerfile package installs PKGLIST="python3 python3-pip virtualenv python3-venv curl" # For the Creality OS, we only need to install these. # We don't override the default name, since that's used by the Moonraker installer # Note that we DON'T want to use the same name as above (not even in this comment) because some parsers might find it. # Note we exclude virtualenv python3-venv curl because they can't be installed on the sonic pad via the package manager. -CREALITY_DEP_LIST="python3 python3-pip" - +CREALITY_DEP_LIST="python3 python3-pip python3-pillow" +SONIC_PAD_DEP_LIST="python3 python3-pip" # # Console Write Helpers @@ -241,7 +242,7 @@ install_or_update_system_dependencies() # The sonic pad always has opkg installed, so we can make sure these packages are installed. # We have had users report issues where this install gets stuck, using the no cache dir flag seems to fix it. opkg update || true - opkg install ${CREALITY_DEP_LIST} || true + opkg install ${SONIC_PAD_DEP_LIST} || true pip3 install -q --no-cache-dir virtualenv else # It seems a lot of printer control systems don't have the date and time set correctly, and then the fail @@ -266,6 +267,9 @@ install_or_update_system_dependencies() log_info "Ensuring zlib is install for Pillow, it's ok if this package install fails." sudo apt install --yes zlib1g-dev 2> /dev/null || true sudo apt install --yes zlib-devel 2> /dev/null || true + sudo apt install --yes python-imaging 2> /dev/null || true + sudo apt install --yes python3-pil 2> /dev/null || true + sudo apt install --yes python3-pillow 2> /dev/null || true fi log_info "System package install complete." From 6f1c5223c7f1a661b80b8d15bee29ff186e5b6d0 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Tue, 21 May 2024 18:00:48 -0700 Subject: [PATCH 092/328] Readme updates --- README.md | 29 +++++++++++++++++------------ docker-readme.md | 4 ++-- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index a00eaf9..e2355bd 100644 --- a/README.md +++ b/README.md @@ -1,38 +1,43 @@

OctoEverywhere's Logo

OctoEverywhere

-Cloud empower your OctoPrint and Klipper printers with **free, private, and unlimited remote access to your full web control portal from anywhere!** Developed for the maker community, powered by the maker community. +Cloud empower your [OctoPrint](https://octoeverywhere.com/?source=github_readme), [Klipper](https://octoeverywhere.com/klipper?source=github_readme), and [Bambu Lab](https://octoeverywhere.com/bambu?source=github_readme) 3D printers with **free, private, and unlimited remote access, AI print failure detection, and more!** Developed for the maker community, powered by the maker community. ## Features -- 🚀 Free remote access to your full OctoPrint, Mainsail, and Fluidd web portals from anywhere. -- 🤖 **Free & unlimited AI failure detection** that will automatically stop failed prints to save you time and money. +- 🚀 Free remote access to your full OctoPrint, Mainsail, Fluidd, and Bambu Lab web portals from anywhere. +- 🤖 **Free & unlimited [AI failure detection](https://octoeverywhere.com/gadget?source=github_readme)** that will automatically stop failed prints to save you time and money. - 📷 Full resolution and full frame-rate webcam streaming. - 📱 Empower your favorite OctoPrint & Klipper iOS and Android apps with [remote access](https://octoeverywhere.com/appsetup?source=github_readme). -- 📺 Live stream your 3D prints to your friends or the entire world with [Live Links](https://octoeverywhere.com/live?source=github_readme). +- 📺 Live stream your 3D prints to your friends or the world with [Live Links](https://octoeverywhere.com/live?source=github_readme). - 🔔 Instant [printer notifications](https://octoeverywhere.com/notifications?source=github_readme) sent to SMS, Email, Discord, Telegram, Slack, and more. -- 🔗 Share secure access of your full OctoPrint portal with others. +- 🔗 Share secure access to your OctoPrint, Fluidd, or Mainsail portal with others. - 💪 Full OctoPrint plugin functionality. - 🤹 Full multicam support. - ... and much, much more! ## Try It Now! -With a Trustpilot rating of **[4.9/5 stars](https://www.trustpilot.com/review/octoeverywhere.com)** and over **96k** makers are already using OctoEverywhere, what are you waiting for? +With a Trustpilot rating of **[4.9/5 stars](https://www.trustpilot.com/review/octoeverywhere.com)** and over **96k** makers already using OctoEverywhere, what are you waiting for?

-**[Click Here To Try OctoEverywhere Now! It's free and takes less than 20 seconds to setup!](https://octoeverywhere.com/getstarted?source=github_plugin_repo)** +**[Click Here To Try OctoEverywhere Now! It's free and takes less than 20 seconds to set up!](https://octoeverywhere.com/getstarted?source=github_readme)**

+## Bambu Connect Docker Container + +If you're using [Bambu Connect](https://octoeverywhere.com/bambu?source=github_readme) to connect a Bambu Lab 3D printer, you can either use our [installer to run directly on any Debian based Linux device,](https://octoeverywhere.com/getstarted?bambu=t&source=github_readme) or you can use our [Docker container to run Bambu Connect](https://blog.octoeverywhere.com/setup-bambu-connect-with-docker-or-docker-compose) on any Windows, Mac, or Linux device. + +## OctoPrint And Klipper Plugin Setup + +Follow our [Getting Started Guide](https://octoeverywhere.com/getstarted?source=github_readme) to get up and running in less than 20 seconds. + ## Bugs & Feedback -We love to hear from you! Please submit bugs or feedback on this github page, our [Discord server](https://discord.gg/v3qbxPee4E), or via [our support system](https://octoeverywhere.com/support). +We love to hear from you! Please submit bugs or feedback on our [Discord server](https://discord.gg/v3qbxPee4E) or via [our support system](https://octoeverywhere.com/support). ## Contributing -Feel free to fork, hack, slash, and PR code! OctoEverywhere is made for the maker community, we appreciate any ideas or help we can get! - -## OctoPrint And Klipper Plugin Setup +Feel free to fork, hack, slash, and PR code! OctoEverywhere is made for the maker community; we appreciate any ideas or help we can get! -Follow our [Getting Started Guide](https://octoeverywhere.com/getstarted?source=github_plugin_repo) to get up and running in less than 20 seconds! \ No newline at end of file diff --git a/docker-readme.md b/docker-readme.md index a8baa12..a43a7fb 100644 --- a/docker-readme.md +++ b/docker-readme.md @@ -27,7 +27,7 @@ You must map the `/data` folder in your docker container to a directory on your Once the docker container is running, you need to look at the logs to find the linking URL. Docker Compose: -`docker-compose logs | grep https://octoeverywhere.com/getstarted` +`docker compose logs | grep https://octoeverywhere.com/getstarted` Docker: `docker logs bambu-connect | grep https://octoeverywhere.com/getstarted` @@ -41,7 +41,7 @@ Using docker compose is the easiest way to run OctoEverywhere's Bambu Connect us - Install [Docker and Docker Compose](https://docs.docker.com/compose/install/linux/) - Clone this repo - Edit the `./docker-compose.yml` file to enter your environment vars -- Run `docker-compose up -d` +- Run `docker compose up -d` - Follow the "Linking Your Bambu Connect Plugin" to link the plugin to your account. ## Using Docker From e631ec4bb75d576e7e9e8df51f6c11676a3a1b00 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Fri, 24 May 2024 14:04:01 -0700 Subject: [PATCH 093/328] Adding a little better logging in the Bambu Connect plugin so if the SN is wrong, the user can know to fix it. --- bambu_octoeverywhere/bambuclient.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/bambu_octoeverywhere/bambuclient.py b/bambu_octoeverywhere/bambuclient.py index bed8d0d..893f85f 100644 --- a/bambu_octoeverywhere/bambuclient.py +++ b/bambu_octoeverywhere/bambuclient.py @@ -44,6 +44,7 @@ def __init__(self, logger:logging.Logger, config:Config, stateTranslator) -> Non self.Version:BambuVersion = None self.HasDoneFirstFullStateSync = False self.ReportSubscribeMid = None + self.IsPendingSubscribe = False self._CleanupStateOnDisconnect() # Get the required args. @@ -183,6 +184,7 @@ def _CleanupStateOnDisconnect(self): self.Version = None self.HasDoneFirstFullStateSync = False self.ReportSubscribeMid = None + self.IsPendingSubscribe = False # Fired when the MQTT connection is made. @@ -192,15 +194,24 @@ def _OnConnect(self, client:mqtt.Client, userdata, flags, reason_code, propertie # We must do this before anything else, otherwise we won't get responses for things like # the full state sync. The result of the subscribe will be reported to _OnSubscribe # Note that at least for my P1P, if the SN is incorrect, the MQTT connection is closed with no _OnSubscribe callback. + # Thus we set the self.IsPendingSubscribe flag, so we can give the user a better error message. + self.IsPendingSubscribe = True (result, self.ReportSubscribeMid) = self.Client.subscribe(f"device/{self.PrinterSn}/report") if result != mqtt.MQTT_ERR_SUCCESS or self.ReportSubscribeMid is None: # If we can't sub, disconnect, since we can't do anything. + self.Logger.warn(f"Failed to subscribe to the MQTT subscription using the serial number '{self.PrinterSn}'. Result: {result}. Disconnecting.") self.Client.disconnect() # Fired when the MQTT connection is lost def _OnDisconnect(self, client, userdata, disconnect_flags, reason_code, properties): - self.Logger.warn("Bambu printer connection lost. We will try to reconnect in a few seconds.") + # If the serial number is wrong in the subscribe call, instead of returning an error the Bambu Lab printers just disconnect. + # So if we were pending a subscribe call, give the user a better error message so they know the likely cause. + if self.IsPendingSubscribe: + self.Logger.error("Bambu printer mqtt connection lost when trying to sub for events.") + self.Logger.error(f"THIS USUALLY MEANS THE PRINTER SERIAL NUMBER IS WRONG. We tried to use the serial number '{self.PrinterSn}'. Double check the SN is correct.") + else: + self.Logger.warn("Bambu printer connection lost. We will try to reconnect in a few seconds.") # Clear the state since we lost the connection and won't stay synced. self._CleanupStateOnDisconnect() From 72f04367612b6998ff82a8754840df3b6f59324d Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sat, 25 May 2024 08:47:52 -0700 Subject: [PATCH 094/328] Adding a few features to the Bambu Connect installer. --- .vscode/launch.json | 2 +- linux_host/networksearch.py | 78 ++++++++++++++++--- .../NetworkConnectors/BambuConnector.py | 33 +++++++- 3 files changed, 98 insertions(+), 15 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 2851e15..c54e0bb 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -119,7 +119,7 @@ // The module requires this json object to be passed. // Normally the install.sh script runs, ensure everything is installed, creates a virtural env, and then runs this modlue giving it these args. // But for debugging, we can skip that assuming it's already been ran. - "{\"OE_REPO_DIR\":\"/home/pi/octoeverywhere\",\"OE_ENV\":\"/home/pi/octoeverywhere-env\",\"USERNAME\":\"pi\",\"USER_HOME\":\"/home/pi\",\"CMD_LINE_ARGS\":\"-skipsudoactions -bambu\"}" + "{\"OE_REPO_DIR\":\"/home/pi/octoeverywhere\",\"OE_ENV\":\"/home/pi/octoeverywhere-env\",\"USERNAME\":\"pi\",\"USER_HOME\":\"/home/pi\",\"CMD_LINE_ARGS\":\"-skipsudoactions -bambu -debug\"}" ] }, { diff --git a/linux_host/networksearch.py b/linux_host/networksearch.py index 87660dd..f379293 100644 --- a/linux_host/networksearch.py +++ b/linux_host/networksearch.py @@ -36,7 +36,8 @@ class NetworkSearch: def ScanForInstances_Bambu(logger:logging.Logger, accessCode:str, printerSn:str, portStr:str = None) -> List[str]: def callback(ip:str): return NetworkSearch.ValidateConnection_Bambu(logger, ip, accessCode, printerSn, portStr, timeoutSec=5) - return NetworkSearch._ScanForInstances(logger, callback) + # We want to return if any one IP is found, since there can only be one printer that will match the printer 100% correct. + return NetworkSearch._ScanForInstances(logger, callback, returnAfterNumberFound=1) # The final two steps can happen in different orders, so we need to wait for both the sub success and state object to be received. @@ -190,7 +191,7 @@ def message(client, userdata:dict, mqttMsg:mqtt.MQTTMessage): # testConFunction must be a function func(ip:str) -> NetworkValidationResult # Returns a list of IPs that reported Success() == True @staticmethod - def _ScanForInstances(logger:logging.Logger, testConFunction) -> List[str]: + def _ScanForInstances(logger:logging.Logger, testConFunction, returnAfterNumberFound = 0) -> List[str]: foundIps = [] try: localIp = NetworkSearch._TryToGetLocalIp() @@ -207,25 +208,78 @@ def _ScanForInstances(logger:logging.Logger, testConFunction) -> List[str]: return foundIps ipPrefix = localIp[:lastDot+1] + # In the past, we did this wide with 255 threads. + # We got some feedback that the system was hanging on lower powered systems, but then I also found a bug where + # if an exception was thrown in the thread, it would hang the system. + # I fixed that but also lowered the concurrent thread count to 100, which seems more comfortable. + totalThreads = 100 + outstandingIpsToCheck = [] counter = 0 + while counter < 255: + # The first IP will be 1, the last 255 + counter += 1 + outstandingIpsToCheck.append(ipPrefix + str(counter)) + + # Start the threads + # We must use arrays so they get captured by ref in the threads. doneThreads = [0] - totalThreads = 255 + hasFoundRequestedNumberOfIps = [False] threadLock = threading.Lock() doneEvent = threading.Event() - while counter <= totalThreads: - fullIp = ipPrefix + str(counter) - def threadFunc(ip): + counter = 0 + while counter < totalThreads: + def threadFunc(threadId): try: - result = testConFunction(ip) + # Loop until we run out of IPs or the test is done by the bool flag. + while True: + # Get the next IP + ip = "none" + with threadLock: + # If there are no IPs left, this thread is done. + if len(outstandingIpsToCheck) == 0: + # This will invoke the finally block. + return + # If enough IPs have been found, we are done. + if hasFoundRequestedNumberOfIps[0] is True: + return + # Get the next IP. + ip = outstandingIpsToCheck.pop() + + # Outside of lock, test the IP + result = testConFunction(ip) + + # re-lock and set the result. + with threadLock: + # If successful, add the IP to the found list. + if result.Success(): + # Enure we haven't already found the requested number of IPs, + # because then the result list might have already been returned + # and we don't want to mess with it. + if hasFoundRequestedNumberOfIps[0] is True: + return + + # Add the IP to the list + foundIps.append(ip) + + # Test if we have found all of the IPs we wanted to find. + if returnAfterNumberFound != 0 and len(foundIps) >= returnAfterNumberFound: + hasFoundRequestedNumberOfIps[0] = True + # We set this now, which allows the function to return the result list + # but the other threads will run until the current test ip is done. + # That's ok since we protect the result list from being added to. + doneEvent.set() + except Exception as e: + # Report the error. + logger.error(f"Server scan failed for {ip} "+str(e)) + finally: + # Important - when we leave for any reason, mark this thread done. with threadLock: - if result.Success(): - foundIps.append(ip) doneThreads[0] += 1 + logger.debug(f"Thread {threadId} done. Done: {doneThreads[0]}; Total: {totalThreads}") + # If all of the threads are done, we are done. if doneThreads[0] == totalThreads: doneEvent.set() - except Exception as e: - logger.error(f"Server scan failed for {ip} "+str(e)) - t = threading.Thread(target=threadFunc, args=[fullIp]) + t = threading.Thread(target=threadFunc, args=(counter,)) t.start() counter += 1 doneEvent.wait() diff --git a/py_installer/NetworkConnectors/BambuConnector.py b/py_installer/NetworkConnectors/BambuConnector.py index 8e1a6a5..f6ff9da 100644 --- a/py_installer/NetworkConnectors/BambuConnector.py +++ b/py_installer/NetworkConnectors/BambuConnector.py @@ -34,7 +34,7 @@ def EnsureBambuConnection(self, context:Context): Logger.Info(f"Keeping the existing Bambu Lab printer connection setup. {ip} - {printerSn}") return - ipOrHostname, port, accessToken, printerSn = self._SetupNewBambuConnection() + ipOrHostname, port, accessToken, printerSn = self._SetupNewBambuConnection(context) Logger.Info(f"You Bambu printer was found and authentication was successful! IP: {ipOrHostname}") # Ensure the X1 camera is setup. @@ -49,7 +49,7 @@ def EnsureBambuConnection(self, context:Context): # Helps the user setup a bambu connection via auto scanning or manual setup. # Returns (ip:str, port:str, accessToken:str, printerSn:str) - def _SetupNewBambuConnection(self): + def _SetupNewBambuConnection(self, context:Context): while True: Logger.Blank() Logger.Blank() @@ -62,6 +62,9 @@ def _SetupNewBambuConnection(self): Logger.Info("Bambu Connect needs your printer's Access Code and Serial Number to connect to your printer.") Logger.Info("If you have any trouble, we are happy to help! Contact us at support@octoeverywhere.com") + # Try to get an an existing access code or SN, so the user doesn't have to re-enter them if they are already there. + oldConfigAccessCode, oldConfigPrinterSn = ConfigHelper.TryToGetBambuData(context) + # Get the access code. accessCode = None while True: @@ -75,6 +78,19 @@ def _SetupNewBambuConnection(self): Logger.Blank() Logger.Info("The access code is case sensitive - make sure to enter it exactly as shown on your printer.") Logger.Blank() + + # If there is already an access code, ask if the user wants to use it. + if oldConfigAccessCode is not None and len(oldConfigAccessCode) > 0: + Logger.Info(f"Your previously entered Access Code is: '{oldConfigAccessCode}'") + if Util.AskYesOrNoQuestion("Do you want to continue using this Access Code?"): + accessCode = oldConfigAccessCode + break + # Set it to None so we wont ask again. + oldConfigAccessCode = None + Logger.Blank() + Logger.Blank() + + # Ask for the access code. accessCode = input("Enter your printer's Access Code: ") # Validate @@ -112,6 +128,19 @@ def _SetupNewBambuConnection(self): Logger.Warn("Follow this link for a step-by-step guide to find the Serial Number for your printer:") Logger.Warn("https://octoeverywhere.com/s/bambu-sn") Logger.Blank() + + # If there is already an sn, ask if the user wants to use it. + if oldConfigPrinterSn is not None and len(oldConfigPrinterSn) > 0: + Logger.Info(f"Your previously entered Serial Number is: '{oldConfigPrinterSn}'") + if Util.AskYesOrNoQuestion("Do you want to continue using this Serial Number?"): + printerSn = oldConfigPrinterSn + break + # Set it to None so we wont ask again. + oldConfigPrinterSn = None + Logger.Blank() + Logger.Blank() + + # Ask for the sn. printerSn = input("Enter your printer's Serial Number: ") # The SN should always be upper case letters. From ddaa7ab9b432d5c6cf25570dc319af1c40ed1e96 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sat, 25 May 2024 09:02:06 -0700 Subject: [PATCH 095/328] Adding more debug logging for MQTT --- bambu_octoeverywhere/bambuclient.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bambu_octoeverywhere/bambuclient.py b/bambu_octoeverywhere/bambuclient.py index 893f85f..91d55f0 100644 --- a/bambu_octoeverywhere/bambuclient.py +++ b/bambu_octoeverywhere/bambuclient.py @@ -225,6 +225,13 @@ def _OnLog(self, client, userdata, level:mqtt.LOGGING_LEVEL, msg:str): Sentry.Exception("MQTT leaked exception.", Exception(msg)) else: self.Logger.error(f"MQTT log error: {msg}") + elif level == mqtt.MQTT_LOG_WARNING: + # Report warnings. + self.Logger.error(f"MQTT log warn: {msg}") + else: + # Report everything else if debug is enabled. + if self.Logger.isEnabledFor(logging.DEBUG): + self.Logger.debug(f"MQTT log: {msg}") # Fried when the MQTT subscribe result has come back. From 17be0747c351667f295079276378a48b861d8752 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sat, 25 May 2024 09:04:39 -0700 Subject: [PATCH 096/328] Debug is too chatty --- bambu_octoeverywhere/bambuclient.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bambu_octoeverywhere/bambuclient.py b/bambu_octoeverywhere/bambuclient.py index 91d55f0..c287c89 100644 --- a/bambu_octoeverywhere/bambuclient.py +++ b/bambu_octoeverywhere/bambuclient.py @@ -228,10 +228,10 @@ def _OnLog(self, client, userdata, level:mqtt.LOGGING_LEVEL, msg:str): elif level == mqtt.MQTT_LOG_WARNING: # Report warnings. self.Logger.error(f"MQTT log warn: {msg}") - else: - # Report everything else if debug is enabled. - if self.Logger.isEnabledFor(logging.DEBUG): - self.Logger.debug(f"MQTT log: {msg}") + # else: + # # Report everything else if debug is enabled. + # if self.Logger.isEnabledFor(logging.DEBUG): + # self.Logger.debug(f"MQTT log: {msg}") # Fried when the MQTT subscribe result has come back. From 0973d16d91e46368c9181b47eb3907f816b51bf6 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sun, 26 May 2024 10:34:19 -0700 Subject: [PATCH 097/328] Updating the Moonraker webcam config logic to handle older Fluidd installs and fixing up a few things. --- .vscode/settings.json | 1 + moonraker_octoeverywhere/moonrakerclient.py | 6 + .../moonrakerwebcamhelper.py | 342 +++++++++++++----- 3 files changed, 249 insertions(+), 100 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index f020be3..3b30af0 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -32,6 +32,7 @@ "Commonize", "comms", "companionconfigfile", + "coms", "continuousprint", "Creality", "Creality's", diff --git a/moonraker_octoeverywhere/moonrakerclient.py b/moonraker_octoeverywhere/moonrakerclient.py index 595e3ca..318a497 100644 --- a/moonraker_octoeverywhere/moonrakerclient.py +++ b/moonraker_octoeverywhere/moonrakerclient.py @@ -28,6 +28,9 @@ class JsonRpcResponse: OE_ERROR_WS_NOT_CONNECTED = 99990001 OE_ERROR_TIMEOUT = 99990002 OE_ERROR_EXCEPTION = 99990003 + # Range helpers. + OE_ERROR_MIN = OE_ERROR_WS_NOT_CONNECTED + OE_ERROR_MAX = OE_ERROR_EXCEPTION def __init__(self, resultObj, errorCode = 0, errorStr : str = None) -> None: self.Result = resultObj @@ -44,6 +47,9 @@ def HasError(self) -> bool: def GetErrorCode(self) -> int: return self.ErrorCode + def IsErrorCodeOeError(self) -> bool: + return self.ErrorCode >= JsonRpcResponse.OE_ERROR_MIN and self.ErrorCode <= JsonRpcResponse.OE_ERROR_MAX + def GetErrorStr(self) -> str: return self.ErrorStr diff --git a/moonraker_octoeverywhere/moonrakerwebcamhelper.py b/moonraker_octoeverywhere/moonrakerwebcamhelper.py index 98b15b5..dfae8c6 100644 --- a/moonraker_octoeverywhere/moonrakerwebcamhelper.py +++ b/moonraker_octoeverywhere/moonrakerwebcamhelper.py @@ -203,60 +203,35 @@ def _DoAutoSettingsUpdate(self): # First, try to use the newer webcam API. # It seems that even if the frontend still uses the older DB based entry, it will still showup in this new API. + # So we do this first, since it's the most correct if it exists. if self._TryToFindWebcamFromApi(): # On success, we found the webcam we want, so we are done. return - # Fallback to the old database system. - # TODO - This should eventually be removed. - result = MoonrakerClient.Get().SendJsonRpcRequest("server.database.get_item", - { - "namespace": "webcams", - } - ) - - # If we failed don't do anything. - if result.HasError(): - if "Namespace 'webcams' not found".lower() in result.ErrorStr.lower(): - # This happens if there are no webcams configured at all. - self.Logger.debug("server.database.get_item returned no webcam namespace found.") - self._ResetValuesToDefaults() - return - self.Logger.warn("Moonraker webcam helper failed to query for webcams. "+result.GetLoggingErrorStr()) + # Fallback to try the old common Moonraker DB webcams + # Note we must keep this around, because some printers are stuck on older version of Moonraker / frontends + # that use these APIs, and they can't be updated. + if self._TryToFindWebcamFromMoonrakerDb(): + # On success, we found the webcam we want, so we are done. return - res = result.GetResult() - if "value" not in res: - self.Logger.warn("Moonraker webcam helper failed to find value in result.") + # Finally fallback to the old way Fluidd stored webcams, in it's own custom db namespace. + # Note we must keep this around, because some printers are stuck on older version of Moonraker / frontends + # that use these APIs, and they can't be updated. + if self._TryToFindWebcamFromFluiddCustomDb(): + # On success, we found the webcam we want, so we are done. return - # To help debugging, log the result. - if self.Logger.isEnabledFor(logging.DEBUG): - self.Logger.debug("Returned webcam database data: %s", json.dumps(res, indent=4, separators=(", ", ": "))) - - value = res["value"] - if len(value) > 0: - # Parse everything we got back. - webcamSettingItems = [] - for guid in value: - webcamSettingsObj = value[guid] - webcamSettings = self._TryToParseWebcamDbEntry(webcamSettingsObj) - if webcamSettings is not None: - webcamSettingItems.append(webcamSettings) - - # If we found anything, set them! - if len(webcamSettingItems) > 0: - self._SetNewValues(webcamSettingItems) - return - # We failed to find a webcam in the list that's valid or there are no webcams in the list. # Revert to defaults + self.Logger.debug("Failed to find any configured webcams, setting defaults.") self._ResetValuesToDefaults() except Exception as e: Sentry.Exception("Webcam helper - _DoAutoSettingsUpdate exception. ", e) + # Tries to find the webcam config using the new Moonraker webcam APIs. def _TryToFindWebcamFromApi(self) -> bool: # It seems that even if the frontend still uses the older DB based entry, it will still showup in this new API. result = MoonrakerClient.Get().SendJsonRpcRequest("server.webcams.list") @@ -298,16 +273,109 @@ def _TryToFindWebcamFromApi(self) -> bool: return False # Set whatever we found. + self.Logger.debug("Using webcam values found in from the new Webcam APIs") self._SetNewValues(webcamSettingsItemResults) # Return success. return True + # Given a Moonraker webcam API result list item, this will try to parse it. + # If successful, this will return a valid AbstractWebcamSettings object. + # If the parse fails or the params are wrong, this will return None + def _TryToParseWebcamApiItem(self, webcamApiItem) -> WebcamSettingItem: + try: + # This new converged logic is amazing, see this doc for the full schema + # https://moonraker.readthedocs.io/en/latest/web_api/#webcam-apis + + # Skip if it's not set to enabled. + if "enabled" in webcamApiItem and webcamApiItem["enabled"] is False: + return None + + # Parse the settings. + webcamSettings = WebcamSettingItem() + if "name" in webcamApiItem: + webcamSettings.Name = webcamApiItem["name"] + if "stream_url" in webcamApiItem: + webcamSettings.StreamUrl = webcamApiItem["stream_url"] + if "snapshot_url" in webcamApiItem: + webcamSettings.SnapshotUrl = webcamApiItem["snapshot_url"] + if "flip_horizontal" in webcamApiItem: + webcamSettings.FlipH = webcamApiItem["flip_horizontal"] + if "flip_vertical" in webcamApiItem: + webcamSettings.FlipV = webcamApiItem["flip_vertical"] + if "rotation" in webcamApiItem: + webcamSettings.Rotation = webcamApiItem["rotation"] + + # Validate and return if we found good settings. + if self._ValidateAndFixupWebCamSettings(webcamSettings) is False: + return None + + # If the settings are validated, return success! + return webcamSettings + except Exception as e: + Sentry.Exception("Webcam helper _TryToParseWebcamApiItem exception. ", e) + return None + + + # Tries to find the webcam config using the older Moonraker common db entry. + def _TryToFindWebcamFromMoonrakerDb(self) -> bool: + # Query the common moonraker webcam database namespace. + result = MoonrakerClient.Get().SendJsonRpcRequest("server.database.get_item", + { + "namespace": "webcams", + } + ) + + # If we failed don't do anything. + if result.HasError(): + # If the error was due to some issue talking to moonraker, we don't want to rest the webcam config to the defaults, SO WE RETURN True. + # When the connection is re-established, we will force sync the webcam settings. + if result.IsErrorCodeOeError(): + self.Logger.warn("Moonraker webcam helper failed to query DB for webcams due to a coms issue. We won't reset the config. "+result.GetLoggingErrorStr()) + return True + # If this happens, it means there are no webcams configured in this DB entry + if "Namespace 'webcams' not found".lower() in result.ErrorStr.lower(): + self.Logger.debug("server.database.get_item returned no webcam namespace found.") + return False + self.Logger.warn("Moonraker webcam helper failed to query DB for webcams. "+result.GetLoggingErrorStr()) + return False + + res = result.GetResult() + if "value" not in res: + self.Logger.warn("Moonraker webcam helper failed to find value in DB result.") + return False + + # To help debugging, log the result. + if self.Logger.isEnabledFor(logging.DEBUG): + self.Logger.debug("Returned webcam database data: %s", json.dumps(res, indent=4, separators=(", ", ": "))) + + value = res["value"] + if len(value) == 0: + return False + + # Parse everything we got back. + webcamSettingItems = [] + for guid in value: + webcamSettingsObj = value[guid] + webcamSettings = self._TryToParseMoonrakerWebcamDbEntry(webcamSettingsObj) + if webcamSettings is not None: + webcamSettingItems.append(webcamSettings) + + # If we didn't get any webcams, return we didn't find anything. + if len(webcamSettingItems) == 0: + return False + + # Set the webcams we found! + self.Logger.debug("Using webcam values found in from the older moonraker common DB webcam entry") + self._SetNewValues(webcamSettingItems) + return True + + # Given a Moonraker webcam db entry, this will try to parse it. # If successful, this will return a valid WebcamSettingItem object. # If the parse fails or the params are wrong, this will return None - def _TryToParseWebcamDbEntry(self, webcamSettingsObj) -> WebcamSettingItem: + def _TryToParseMoonrakerWebcamDbEntry(self, webcamSettingsObj) -> WebcamSettingItem: try: # Skip if it's not set to enabled. if "enabled" in webcamSettingsObj and webcamSettingsObj["enabled"] is False: @@ -355,32 +423,102 @@ def _TryToParseWebcamDbEntry(self, webcamSettingsObj) -> WebcamSettingItem: return None - # Given a Moonraker webcam API result list item, this will try to parse it. - # If successful, this will return a valid AbstractWebcamSettings object. + # Tries to find the webcam config using the older Fluidd common namespace db entry. + def _TryToFindWebcamFromFluiddCustomDb(self) -> bool: + # Older versions of Fluidd had their own DB entries in a custom namespace. + # The format is something like this: + # https://github.com/fluidd-core/fluidd/blob/8f091c2c75c6646cd29ab288863a379b7ca6c63e/src/store/webcams/actions.ts#L34 + result = MoonrakerClient.Get().SendJsonRpcRequest("server.database.get_item", + { + "namespace": "fluidd", + "key": "cameras" + } + ) + + # If we failed don't do anything. + if result.HasError(): + # If the error was due to some issue talking to moonraker, we don't want to rest the webcam config to the defaults, SO WE RETURN True. + # When the connection is re-established, we will force sync the webcam settings. + if result.IsErrorCodeOeError(): + self.Logger.warn("Moonraker webcam helper failed to query DB for webcams due to a coms issue. We won't reset the config. "+result.GetLoggingErrorStr()) + return True + # If this happens, it means there are no webcams configured in this DB entry + if "not found".lower() in result.ErrorStr.lower(): + self.Logger.debug("server.database.get_item for custom FLUIDD namespace returned no webcam namespace found.") + return False + self.Logger.warn("Moonraker webcam helper failed to FLUIDD DB query for webcams. "+result.GetLoggingErrorStr()) + return False + + res = result.GetResult() + if "value" not in res: + self.Logger.warn("Moonraker webcam helper failed to find value in result FLUIDD DB.") + return False + + # To help debugging, log the result. + if self.Logger.isEnabledFor(logging.DEBUG): + self.Logger.debug("Returned FLUIDD webcam database data: %s", json.dumps(res, indent=4, separators=(", ", ": "))) + + value = res["value"] + if len(value) == 0: + return False + + # Parse everything we got back. + webcamSettingItems = [] + for guid in value: + webcamSettingsObj = value[guid] + webcamSettings = self._TryToParseFluiddCustomWebcamDbEntry(webcamSettingsObj) + if webcamSettings is not None: + webcamSettingItems.append(webcamSettings) + + # If we found anything, set them! + if len(webcamSettingItems) == 0: + return False + + self.Logger.debug("Using webcam values found in from the old FLUIDD custom namespace db entry") + self._SetNewValues(webcamSettingItems) + return True + + + # Given a Fluidd namespace webcam db entry, this will try to parse it. + # If successful, this will return a valid WebcamSettingItem object. # If the parse fails or the params are wrong, this will return None - def _TryToParseWebcamApiItem(self, webcamApiItem) -> WebcamSettingItem: + def _TryToParseFluiddCustomWebcamDbEntry(self, webcamSettingsObj) -> WebcamSettingItem: try: - # This new converged logic is amazing, see this doc for the full schema - # https://moonraker.readthedocs.io/en/latest/web_api/#webcam-apis - # Skip if it's not set to enabled. - if "enabled" in webcamApiItem and webcamApiItem["enabled"] is False: + if "enabled" in webcamSettingsObj and webcamSettingsObj["enabled"] is False: return None - # Parse the settings. webcamSettings = WebcamSettingItem() - if "name" in webcamApiItem: - webcamSettings.Name = webcamApiItem["name"] - if "stream_url" in webcamApiItem: - webcamSettings.StreamUrl = webcamApiItem["stream_url"] - if "snapshot_url" in webcamApiItem: - webcamSettings.SnapshotUrl = webcamApiItem["snapshot_url"] - if "flip_horizontal" in webcamApiItem: - webcamSettings.FlipH = webcamApiItem["flip_horizontal"] - if "flip_vertical" in webcamApiItem: - webcamSettings.FlipV = webcamApiItem["flip_vertical"] - if "rotation" in webcamApiItem: - webcamSettings.Rotation = webcamApiItem["rotation"] + if "name" in webcamSettingsObj: + webcamSettings.Name = webcamSettingsObj["name"] + + # There seems to only be one URL, which can be the snapshot URL or the stream URL. + # Our _ValidateAndFixupWebCamSettings requires a stream url to be set, but it will try to figure out the snapshot URL + # if it's not set. So we will always set the URL for the stream, and sometimes the snapshot. + # Note: It seems that most of the time, this URL is a stream URL. + if "url" in webcamSettingsObj: + url:str = webcamSettingsObj["url"] + + # Try to detect if this is a snapshot or stream URL + urlLower = url.lower() + if urlLower.find("snapshot") != -1: + webcamSettings.SnapshotUrl = url + # For the stream url, always use the basic URL. If we can find "snapshot" replace it with "stream" + # The url might be like .../webcam/snapshot or .../webcam/action=snapshot in which case, stream might work. + webcamSettings.StreamUrl = urlLower.replace("snapshot", "stream") + else: + # Assume it's a stream URL, and try to set it. The validation system will handle getting the snapshot url from it. + webcamSettings.StreamUrl = url + + # Set the flip values. + if "flipX" in webcamSettingsObj: + webcamSettings.FlipH = webcamSettingsObj["flipX"] + if "flipY" in webcamSettingsObj: + webcamSettings.FlipV = webcamSettingsObj["flipY"] + + # Set rotation + if "rotate" in webcamSettingsObj: + webcamSettings.Rotation = webcamSettingsObj["rotate"] # Validate and return if we found good settings. if self._ValidateAndFixupWebCamSettings(webcamSettings) is False: @@ -389,7 +527,7 @@ def _TryToParseWebcamApiItem(self, webcamApiItem) -> WebcamSettingItem: # If the settings are validated, return success! return webcamSettings except Exception as e: - Sentry.Exception("Webcam helper _TryToParseWebcamApiItem exception. ", e) + Sentry.Exception("Webcam helper _TryToParseFluiddCustomWebcamDbEntry exception. ", e) return None @@ -450,50 +588,54 @@ def _ValidateAndFixupWebCamSettings(self, webcamSettings:WebcamSettingItem) -> b return False - # Tries to find the snapshot URL, if it's successful, it returns the url - # If it fails, it return None - def _TryToFigureOutSnapshotUrl(self, streamUrl): + # Tries to find the snapshot URL. + # If successful, it returns the snapshot URL + # If failed, it return None + def _TryToFigureOutSnapshotUrl(self, streamUrl:str) -> str: # If we have no snapshot url, see if we can figure one out. # We know most all webcam interfaces use the "mjpegstreamer" web url signatures. # So if we find "action=stream" as in "http://127.0.0.1/webcam/?action=stream", try to get a snapshot. + # Modern webcam servers also use .../webcam/stream and .../webcam/snapshot, so we will try that as well. + # Update, the easiest way to do this is if we find "stream", just replace it with "snapshot", and see if it still works. + c_streamAction = "stream" + c_snapshotAction = "snapshot" + + # See if stream exists. streamUrlLower = streamUrl.lower() - c_streamAction = "action=stream" - c_snapshotAction = "action=snapshot" - indexOfStreamSuffix = streamUrlLower.find(c_streamAction) - - if indexOfStreamSuffix != -1: - # We found the action=stream, try replacing it and see if we hit a valid endpoint. - # keep the original string around, so we can return it if things work out. - possibleSnapshotUrl = streamUrl[:indexOfStreamSuffix] + c_snapshotAction + streamUrl[indexOfStreamSuffix + len(c_streamAction):] - try: - # Make sure the path is a full URL - # If not, assume localhost port 80. - testSnapshotUrl = possibleSnapshotUrl - if testSnapshotUrl.lower().startswith("http") is False: - if testSnapshotUrl.startswith("/") is False: - testSnapshotUrl = "/"+testSnapshotUrl - testSnapshotUrl = "http://127.0.0.1"+testSnapshotUrl - self.Logger.debug("Trying to find a snapshot url, testing: %s - from stream URL: %s", testSnapshotUrl, streamUrl) - - # We can't use .head because that only pulls the headers from nginx, it doesn't get the full headers. - # So we use .get with a timeout. - with requests.get(testSnapshotUrl, timeout=20) as response: - # Check for success - if response.status_code != 200: - return None - - # This is a good sign, check the content type. - contentTypeHeaderKey = "content-type" - if contentTypeHeaderKey in response.headers: - if "image" in response.headers[contentTypeHeaderKey].lower(): - # Success! - self.Logger.debug("Found a valid snapshot URL! Url: %s, Content-Type: %s", testSnapshotUrl, response.headers[contentTypeHeaderKey]) - return possibleSnapshotUrl - - except Exception: - pass - # On any failure, return None - self.Logger.debug("FAILED to find a snapshot url from stream URL") + if streamUrlLower.find(c_streamAction) == -1: + self.Logger.debug("FAILED to find a snapshot url from stream URL, no stream suffix found.") + return None + + # We found the stream, try replacing it and see if we hit a valid endpoint. + # Use the lower version to ensure the match, the case of a URL shouldn't matter. + possibleSnapshotUrl = streamUrlLower.replace(c_streamAction, c_snapshotAction) + try: + # Make sure the path is a full URL. If not, assume localhost port 80. + absoluteSnapshotUrl = possibleSnapshotUrl + if absoluteSnapshotUrl.lower().startswith("http") is False: + if absoluteSnapshotUrl.startswith("/") is False: + absoluteSnapshotUrl = "/"+absoluteSnapshotUrl + absoluteSnapshotUrl = "http://127.0.0.1"+absoluteSnapshotUrl + self.Logger.debug("Trying to find a snapshot url, testing: %s - from stream URL: %s", absoluteSnapshotUrl, streamUrl) + + # We can't use .head because that only pulls the headers from nginx, it doesn't get the full headers. + # So we use .get with a timeout. + with requests.get(absoluteSnapshotUrl, timeout=20) as response: + # Check for success + if response.status_code != 200: + self.Logger.debug(f"Test snapshot attempt returned http status {response.status_code}. Url: {absoluteSnapshotUrl}") + return None + + # This is a good sign, check the content type. + contentTypeHeaderKey = "content-type" + if contentTypeHeaderKey in response.headers: + if "image" in response.headers[contentTypeHeaderKey].lower(): + # Success! + self.Logger.debug("Found a valid snapshot URL! Url: %s, Content-Type: %s", absoluteSnapshotUrl, response.headers[contentTypeHeaderKey]) + return possibleSnapshotUrl + self.Logger.debug(f"We made the web request for the stream URL but didn't get a valid result. {streamUrl}") + except Exception as e: + self.Logger.debug(f"FAILED to find a snapshot url from stream URL. Url: {streamUrl}, Error: {str(e)}") return None From ecb06ba0a51cabb97d9f5fbbe45664a2cbf4b60a Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Tue, 28 May 2024 22:17:46 -0700 Subject: [PATCH 098/328] Adding some testing logic to test DNS resoultion when the main websocket fails to resolve. --- .vscode/settings.json | 2 ++ octoeverywhere/dnstest.py | 54 +++++++++++++++++++++++++++++++++ octoeverywhere/octoservercon.py | 7 +++++ 3 files changed, 63 insertions(+) create mode 100644 octoeverywhere/dnstest.py diff --git a/.vscode/settings.json b/.vscode/settings.json index 3b30af0..3bac181 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -144,6 +144,7 @@ "octowebstreamhttphelperimpl", "octowebstreamimpl", "octowebstreamwshelper", + "oeapi", "oestreamboundary", "openwrt", "opkg", @@ -207,6 +208,7 @@ "timerprogress", "timesyncd", "toradio", + "trafficmanager", "Trustpilot", "UDISK", "uduuitfqrsstnhhjpsxhmyqwvpxgnajqqbhxferoxunusjaybodfotkupjaecnccdxzwmeajqqmjftnhoonusnjatqcryxfvrzgibouexjflbrmurkhltmsd", diff --git a/octoeverywhere/dnstest.py b/octoeverywhere/dnstest.py new file mode 100644 index 0000000..5e1f869 --- /dev/null +++ b/octoeverywhere/dnstest.py @@ -0,0 +1,54 @@ +import time +import logging + +import dns.resolver + +# Created to the DNS resolution of our URLS when the websocket claims it can't connect due to DNS issues. +class DnsTest: + + def __init__(self, logger: logging.Logger) -> None: + self.Logger = logger + + + def RunTestSync(self) -> None: + try: + self.Logger.info("DNS test starting.") + # Try to get the cname, which should be octoeverywhere-v1.trafficmanager.net + self._TestUrl("starport-v1.octoeverywhere.com", "CNAME") + # Try to resolve the octoeverywhere-v1.trafficmanager.net, which should resolve to a cluster hostname. + self._TestUrl("octoeverywhere-v1.trafficmanager.net", "CNAME") + # Try to resolve the octoeverywhere-v1.trafficmanager.net, which should resolve to a cluster ip. + self._TestUrl("octoeverywhere-v1.trafficmanager.net", "A") + # This should do the same as above and resolve the IP. + self._TestUrl("starport-v1.octoeverywhere.com", "A") + + # For fun, also do the root and some others. + self._TestUrl("octoeverywhere.com", "CNAME") + self._TestUrl("octoeverywhere.com", "A") + self._TestUrl("printer-events-v1-oeapi.octoeverywhere.com") + self._TestUrl("gadget-v1-oeapi.octoeverywhere.com") + + # Also, run some DNS names we expect to be valid. + self._TestUrl("google.com", "A") + self._TestUrl("bing.com", "A") + self.Logger.info("DNS test done.") + except Exception as e: + self.Logger.error(f"RunTestSync test failed. {e}") + + + def _TestUrl(self, url: str, recordType:str = "A") -> None: + try: + self.Logger.debug(f"Starting DNS resolve test for {url} with record type {recordType}") + startSec = time.time() + dnsResolver = dns.resolver.Resolver() + dnsResolver.timeout = 5.0 # Timeout in seconds. + result = dnsResolver.query(url, recordType) + resolveTimeSec = time.time() - startSec + c = 0 + for r in result: + c += 1 + self.Logger.info(f"[{c}/{len(result)}] Resolved {url}:{recordType} to {r} in {resolveTimeSec:.3f} seconds.") + if len(result) == 0: + self.Logger.info(f"[?/?] FAILED TO RESOLVE {url}:{recordType} in {resolveTimeSec:.3f} seconds - no result returned.") + except Exception as e: + self.Logger.info(f"TestUrl test failed. Url: {url}, Error: {e}") diff --git a/octoeverywhere/octoservercon.py b/octoeverywhere/octoservercon.py index 02531c9..be72ac1 100644 --- a/octoeverywhere/octoservercon.py +++ b/octoeverywhere/octoservercon.py @@ -8,6 +8,7 @@ from .repeattimer import RepeatTimer from .octopingpong import OctoPingPong from .threaddebug import ThreadDebug +from .dnstest import DnsTest # # This class is responsible for connecting and maintaining a connection to a server. @@ -152,6 +153,12 @@ def OnError(self, ws, err): self.Logger.info("Blocking lowest latency endpoint, since we failed while the WS connect was happening.") self.Logger.error("OctoEverywhere Ws error: " +str(err)) + # TODO - remove this code after debugging some DNS issues. + # We added this logic to do an active test of the DNS names. + if "failure in name resolution" in str(err): + dnsTest = DnsTest(self.Logger) + dnsTest.RunTestSync() + def OnMsg(self, ws, msg): # When we get any message, consider it user activity. From 0c76a8161e2a10a14c4b85997c12e52960280ce2 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Tue, 28 May 2024 22:19:08 -0700 Subject: [PATCH 099/328] Version bump --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index beb7b68..1a60f4d 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this single version string is used by all of the plugins in OctoEverywhere! -plugin_version = "3.3.4" +plugin_version = "3.3.5" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 58d9fcddd26dab41c5b498c34da2da74ddb3b4b3 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Tue, 28 May 2024 22:22:46 -0700 Subject: [PATCH 100/328] Adding a little more debug logging to the Bambu connect scanner. --- linux_host/networksearch.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/linux_host/networksearch.py b/linux_host/networksearch.py index f379293..55a6ed5 100644 --- a/linux_host/networksearch.py +++ b/linux_host/networksearch.py @@ -147,11 +147,13 @@ def message(client, userdata:dict, mqttMsg:mqtt.MQTTMessage): # Try to connect, this will throw if it fails to find any server to connect to. failedToConnect = True try: + logger.debug(f"Connecting to Bambu on {ipOrHostname}:{port}...") client.connect(ipOrHostname, port, keepalive=60) failedToConnect = False client.loop_start() except Exception as e: logger.debug(f"Bambu {ipOrHostname} - connection failure {e}") + logger.debug(f"Connection exit for Bambu on {ipOrHostname}:{port}") # Wait for the timeout. if not failedToConnect: From f6dbe0938b434e035ec9c76f7730c69eb4ff33a2 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Thu, 30 May 2024 07:55:27 -0700 Subject: [PATCH 101/328] Minor logging change. --- moonraker_octoeverywhere/moonrakerwebcamhelper.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/moonraker_octoeverywhere/moonrakerwebcamhelper.py b/moonraker_octoeverywhere/moonrakerwebcamhelper.py index dfae8c6..fb0f378 100644 --- a/moonraker_octoeverywhere/moonrakerwebcamhelper.py +++ b/moonraker_octoeverywhere/moonrakerwebcamhelper.py @@ -428,6 +428,7 @@ def _TryToFindWebcamFromFluiddCustomDb(self) -> bool: # Older versions of Fluidd had their own DB entries in a custom namespace. # The format is something like this: # https://github.com/fluidd-core/fluidd/blob/8f091c2c75c6646cd29ab288863a379b7ca6c63e/src/store/webcams/actions.ts#L34 + self.Logger.debug("Webcam helper is trying to get webcam settings from the custom fluidd namespace...") result = MoonrakerClient.Get().SendJsonRpcRequest("server.database.get_item", { "namespace": "fluidd", @@ -440,7 +441,7 @@ def _TryToFindWebcamFromFluiddCustomDb(self) -> bool: # If the error was due to some issue talking to moonraker, we don't want to rest the webcam config to the defaults, SO WE RETURN True. # When the connection is re-established, we will force sync the webcam settings. if result.IsErrorCodeOeError(): - self.Logger.warn("Moonraker webcam helper failed to query DB for webcams due to a coms issue. We won't reset the config. "+result.GetLoggingErrorStr()) + self.Logger.warn("Moonraker webcam helper failed to query DB for FLUIDD webcams due to a coms issue. We won't reset the config. "+result.GetLoggingErrorStr()) return True # If this happens, it means there are no webcams configured in this DB entry if "not found".lower() in result.ErrorStr.lower(): @@ -472,6 +473,7 @@ def _TryToFindWebcamFromFluiddCustomDb(self) -> bool: # If we found anything, set them! if len(webcamSettingItems) == 0: + self.Logger.debug("No valid webcam config objects found in the FLUIDD custom namespace db entry.") return False self.Logger.debug("Using webcam values found in from the old FLUIDD custom namespace db entry") From efa9cb7fc033bcf587540c49d582f3f3eb3c04b6 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Fri, 14 Jun 2024 20:44:19 -0700 Subject: [PATCH 102/328] Fixing a small bug not allowing spoolman to work on bambu connect plugins. --- .vscode/settings.json | 1 + octoeverywhere/WebStream/octowebstreamhttphelper.py | 4 +++- octoeverywhere/WebStream/octowebstreamwshelper.py | 9 +++++---- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 3bac181..969513b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -195,6 +195,7 @@ "somename", "sooooooo", "Spammy", + "Spoolman", "sslopt", "Starbound", "starport", diff --git a/octoeverywhere/WebStream/octowebstreamhttphelper.py b/octoeverywhere/WebStream/octowebstreamhttphelper.py index d1af95e..8a5f4cf 100644 --- a/octoeverywhere/WebStream/octowebstreamhttphelper.py +++ b/octoeverywhere/WebStream/octowebstreamhttphelper.py @@ -22,6 +22,7 @@ from ..Proto import DataCompression from ..Proto import MessagePriority from ..Proto import OeAuthAllowed +from ..Proto.PathTypes import PathTypes # # A helper object that handles http request for the web stream system. @@ -172,7 +173,8 @@ def executeHttpRequest(self): octoHttpResult = CommandHandler.Get().HandleCommand(httpInitialContext, self.UploadBuffer) else: # This is a normal web request, first ensure they are allowed. - if OctoHttpRequest.GetDisableHttpRelay(): + # Note we must always allow absolute paths, since these can be services like Spoolman or OctoFarm. + if OctoHttpRequest.GetDisableHttpRelay() and httpInitialContext.PathType() != PathTypes.Absolute: self.Logger.warn("OctoWebStreamHttpHelper got a request but the http relay is disabled.") self.WebStream.SetClosedDueToFailedRequestConnection() self.WebStream.Close() diff --git a/octoeverywhere/WebStream/octowebstreamwshelper.py b/octoeverywhere/WebStream/octowebstreamwshelper.py index 619a2fa..2c9df55 100644 --- a/octoeverywhere/WebStream/octowebstreamwshelper.py +++ b/octoeverywhere/WebStream/octowebstreamwshelper.py @@ -49,15 +49,16 @@ def __init__(self, streamId, logger, webStream, webStreamOpenMsg, openedTime): self.IsWsObjOpened = False self.IsWsObjClosed = False - # Ensure that the http relay is enabled - if OctoHttpRequest.GetDisableHttpRelay(): - raise Exception("Web stream ws was attempted to be started when the http relay is disabled.") - # Capture the initial http context self.HttpInitialContext = webStreamOpenMsg.HttpInitialContext() if self.HttpInitialContext is None: raise Exception("Web stream ws helper got a open message with no http context") + # Ensure that the http relay is enabled + # Note we must always allow absolute paths, since these can be services like Spoolman or OctoFarm. + if OctoHttpRequest.GetDisableHttpRelay() and self.HttpInitialContext.PathType() != PathTypes.PathTypes.Absolute: + raise Exception("Web stream ws was attempted to be started when the http relay is disabled.") + # Parse the headers, filter them, and keep them locally. # This is required for klipper clients, since they need to send the X-API-Key header with the API key. self.Headers = HeaderHelper.GatherWebsocketRequestHeaders(self.Logger, self.HttpInitialContext) From e1bf31bbc535b7896315ce3f3302091934564f29 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Fri, 14 Jun 2024 20:45:09 -0700 Subject: [PATCH 103/328] Version bump. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1a60f4d..10b25e6 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this single version string is used by all of the plugins in OctoEverywhere! -plugin_version = "3.3.5" +plugin_version = "3.3.6" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 87719f8239841b1b9c85db4640f048a246651ecd Mon Sep 17 00:00:00 2001 From: Alec <30310787+alec-pinson@users.noreply.github.com> Date: Sat, 15 Jun 2024 21:09:29 +0100 Subject: [PATCH 104/328] Publish version tag + docker image tidy (#66) * Publish version tag + docker image tidy * forgot to revert these bits sorry * 1 more * update comment --- .github/workflows/docker-publish.yml | 8 +++++++- Dockerfile | 9 ++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 6fd5da9..aa77ed4 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -38,6 +38,12 @@ jobs: uses: docker/metadata-action@v5 with: images: octoeverywhere/octoeverywhere + tags: | + # set latest tag + type=raw,value=latest + # set versioned tag + type=semver,pattern={{version}} + - name: Build and push Docker image id: push @@ -48,7 +54,7 @@ jobs: platforms: linux/amd64,linux/arm/v7,linux/arm64 file: ./Dockerfile push: true - tags: octoeverywhere/octoeverywhere:latest + tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} # This isn't working, so it's disabled for now. diff --git a/Dockerfile b/Dockerfile index f5f99dd..dbfde98 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ -# Start with the lastest ubuntu, for a solid base, +# Start with the lastest alpine, for a solid base, # since we need some advance binaries for things like pillow and ffmpeg. -FROM ubuntu:latest +FROM alpine:3.20.0 # We will base ourselves in root, becuase why not. WORKDIR /root @@ -16,8 +16,7 @@ ENV DATA_DIR=/data/ # Install the required packages. # Any packages here should be mirrored in the install script - and any optaionl pillow packages done inline. -RUN apt update -RUN apt install -y curl ffmpeg jq python3 python3-pip python3-venv virtualenv libjpeg-dev zlib1g-dev python3-pil python3-pillow +RUN apk add --no-cache curl ffmpeg jq python3 py3-pip py3-virtualenv jpeg-dev libjpeg-turbo-dev zlib-dev py3-pillow # # We decided to not run the installer, since the point of the installer is to setup the env, build the launch args, and setup the serivce. @@ -34,4 +33,4 @@ RUN ${VENV_DIR}/bin/pip3 install --require-virtualenv --no-cache-dir -q -r ${REP # For docker, we use our docker_octoeverywhere host to handle the runtime setup and launch of the serivce. WORKDIR ${REPO_DIR} # Use the full path to the venv, we msut use this [] notation for our ctlc handler to work in the contianer -ENTRYPOINT ["/root/octoeverywhere-env/bin/python", "-m", "docker_octoeverywhere"] \ No newline at end of file +ENTRYPOINT ["/root/octoeverywhere-env/bin/python", "-m", "docker_octoeverywhere"] From dab33d287029a9b225f5879fd9fada4b1a6d1e51 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sat, 15 Jun 2024 13:38:27 -0700 Subject: [PATCH 105/328] Fixing the ARM docker container by adding required deps. --- .github/workflows/docker-publish.yml | 1 - Dockerfile | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index aa77ed4..5073606 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -50,7 +50,6 @@ jobs: uses: docker/build-push-action@v5 with: context: . - # These are the platform our ubuntu image base supports platforms: linux/amd64,linux/arm/v7,linux/arm64 file: ./Dockerfile push: true diff --git a/Dockerfile b/Dockerfile index dbfde98..132cc58 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,8 @@ ENV DATA_DIR=/data/ # Install the required packages. # Any packages here should be mirrored in the install script - and any optaionl pillow packages done inline. -RUN apk add --no-cache curl ffmpeg jq python3 py3-pip py3-virtualenv jpeg-dev libjpeg-turbo-dev zlib-dev py3-pillow +# GCC, python3-dev, and musl-dev are required for pillow, and jpeg-dev and zlib-dev are required for jpeg support. +RUN apk add --no-cache curl ffmpeg jq python3 python3-dev gcc musl-dev py3-pip py3-virtualenv jpeg-dev libjpeg-turbo-dev zlib-dev py3-pillow # # We decided to not run the installer, since the point of the installer is to setup the env, build the launch args, and setup the serivce. From 6080c8e7ea96fa80cc5dff7dc56583b2aa9c34b0 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sat, 15 Jun 2024 13:38:46 -0700 Subject: [PATCH 106/328] Version bump. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 10b25e6..55f33c1 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this single version string is used by all of the plugins in OctoEverywhere! -plugin_version = "3.3.6" +plugin_version = "3.3.7" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 415ed1a442f26a45c55e973602e1e6aa1d9f239c Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Wed, 19 Jun 2024 07:55:02 -0700 Subject: [PATCH 107/328] Adding a zstandard as a new optional protocol compression option! --- .github/workflows/pylint.yml | 5 + .vscode/settings.json | 17 +- Dockerfile | 5 + bambu_octoeverywhere/bambuhost.py | 4 + developer/devnotes.md | 2 +- moonraker_octoeverywhere/moonrakerhost.py | 4 + octoeverywhere/Proto/DataCompression.py | 1 + octoeverywhere/Proto/HandshakeSyn.py | 15 +- octoeverywhere/WebStream/octowebstream.py | 8 +- .../WebStream/octowebstreamhttphelper.py | 230 ++++---- .../WebStream/octowebstreamwshelper.py | 44 +- octoeverywhere/commandhandler.py | 6 +- octoeverywhere/compression.py | 493 ++++++++++++++++++ octoeverywhere/octohttprequest.py | 24 +- octoeverywhere/octosessionimpl.py | 10 +- octoeverywhere/octostreammsgbuilder.py | 4 +- octoeverywhere/zstandarddictionary.py | 134 +++++ octoprint_octoeverywhere/__init__.py | 4 + octoprint_octoeverywhere/__main__.py | 10 +- .../octoprintwebcamhelper.py | 2 +- octoprint_octoeverywhere/slipstream.py | 29 +- py_installer/Installer.py | 8 + py_installer/Updater.py | 8 + py_installer/ZStandard.py | 78 +++ requirements.txt | 1 + setup.py | 9 +- 26 files changed, 985 insertions(+), 170 deletions(-) create mode 100644 octoeverywhere/compression.py create mode 100644 octoeverywhere/zstandarddictionary.py create mode 100644 py_installer/ZStandard.py diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index cf93885..85b2ce2 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -16,12 +16,17 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} + - name: Install dependencies + # We always install zstandard by hand, since it's an optional lib. + # Ideally this version will stay in sync with Compression.ZStandardPipPackageString run: | python -m pip install --upgrade pip pip install pylint pip install octoprint pip install -r requirements.txt + pip install "zstandard>=0.21.0,<0.23.0" + - name: Analysing the code with pylint run: | pylint ./octoeverywhere/ diff --git a/.vscode/settings.json b/.vscode/settings.json index 969513b..f5e48b6 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,4 @@ { - "python.pythonPath": "py3venv/bin/python", "cSpell.words": [ "ABNF", "accesscontrol", @@ -41,6 +40,7 @@ "crowsnest", "crypo", "Damerell", + "decompressor", "deps", "devel", "devs", @@ -64,6 +64,7 @@ "frontendsetup", "fsensor", "fstring", + "FULLDICT", "gcode", "geteuid", "getpwnam", @@ -97,6 +98,7 @@ "localip", "Mailsail", "mainsailconfighandler", + "mbps", "mdns", "microreads", "mjpeg", @@ -156,6 +158,7 @@ "permissioned", "PKGLIST", "platformcompat", + "precompute", "printerevent", "printerid", "printernotifications", @@ -245,15 +248,13 @@ "zhops", "zmoves", "zoffset", + "zstandard", + "zstandarddictionary", "zvalue" ], - "python.linting.pylintEnabled": true, - "python.linting.enabled": true, - "python.linting.pylintArgs": [ - "--rcfile", - "${workspaceFolder}/.pylintrc" - ], "cSpell.enableFiletypes": [ "shellscript" - ] + ], + "pylint.showNotification": true, + "pylint.lintOnChange": true, } \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 132cc58..f32011c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,6 +31,11 @@ RUN ${VENV_DIR}/bin/python -m pip install --upgrade pip COPY ./ ${REPO_DIR}/ RUN ${VENV_DIR}/bin/pip3 install --require-virtualenv --no-cache-dir -q -r ${REPO_DIR}/requirements.txt +# Install the optional pacakges for zstandard compression. +# THIS VERSION STRING MUST STAY IN SYNC with Compression.ZStandardPipPackageString +RUN apk add zstd +RUN ${VENV_DIR}/bin/pip3 install --require-virtualenv --no-cache-dir -q "zstandard>=0.21.0,<0.23.0" + # For docker, we use our docker_octoeverywhere host to handle the runtime setup and launch of the serivce. WORKDIR ${REPO_DIR} # Use the full path to the venv, we msut use this [] notation for our ctlc handler to work in the contianer diff --git a/bambu_octoeverywhere/bambuhost.py b/bambu_octoeverywhere/bambuhost.py index 6370d09..104ecbe 100644 --- a/bambu_octoeverywhere/bambuhost.py +++ b/bambu_octoeverywhere/bambuhost.py @@ -5,6 +5,7 @@ from octoeverywhere.sentry import Sentry from octoeverywhere.telemetry import Telemetry from octoeverywhere.hostcommon import HostCommon +from octoeverywhere.compression import Compression from octoeverywhere.printinfo import PrintInfoManager from octoeverywhere.Webcam.webcamhelper import WebcamHelper from octoeverywhere.octopingpong import OctoPingPong @@ -95,6 +96,9 @@ def RunBlocking(self, configPath, localStorageDir, repoRoot, devConfig_CanBeNone if DevLocalServerAddress_CanBeNone is not None: Telemetry.SetServerProtocolAndDomain("http://"+DevLocalServerAddress_CanBeNone) + # Init compression + Compression.Init(self.Logger, localStorageDir) + # Init the mdns client MDns.Init(self.Logger, localStorageDir) diff --git a/developer/devnotes.md b/developer/devnotes.md index da70b9c..da10455 100644 --- a/developer/devnotes.md +++ b/developer/devnotes.md @@ -27,7 +27,7 @@ ## Install Other Branches: - - https://github.com/QuinnDamerell/OctoPrint-OctoEverywhere/archive/ui.zip + - https://github.com/QuinnDamerell/OctoPrint-OctoEverywhere/archive/compress.zip ## Before checking in: - Run in py2 env diff --git a/moonraker_octoeverywhere/moonrakerhost.py b/moonraker_octoeverywhere/moonrakerhost.py index 74b203e..7fc922d 100644 --- a/moonraker_octoeverywhere/moonrakerhost.py +++ b/moonraker_octoeverywhere/moonrakerhost.py @@ -5,6 +5,7 @@ from octoeverywhere.sentry import Sentry from octoeverywhere.telemetry import Telemetry from octoeverywhere.hostcommon import HostCommon +from octoeverywhere.compression import Compression from octoeverywhere.octopingpong import OctoPingPong from octoeverywhere.Webcam.webcamhelper import WebcamHelper from octoeverywhere.printinfo import PrintInfoManager @@ -120,6 +121,9 @@ def RunBlocking(self, klipperConfigDir, isCompanionMode, localStorageDir, servic if DevLocalServerAddress_CanBeNone is not None: Telemetry.SetServerProtocolAndDomain("http://"+DevLocalServerAddress_CanBeNone) + # Init compression + Compression.Init(self.Logger, localStorageDir) + # Init the mdns client MDns.Init(self.Logger, localStorageDir) diff --git a/octoeverywhere/Proto/DataCompression.py b/octoeverywhere/Proto/DataCompression.py index 283c213..c4665ba 100644 --- a/octoeverywhere/Proto/DataCompression.py +++ b/octoeverywhere/Proto/DataCompression.py @@ -6,3 +6,4 @@ class DataCompression(object): None_ = 0 Brotli = 1 Zlib = 2 + ZStandard = 3 diff --git a/octoeverywhere/Proto/HandshakeSyn.py b/octoeverywhere/Proto/HandshakeSyn.py index 4dd19e6..ddf0c56 100644 --- a/octoeverywhere/Proto/HandshakeSyn.py +++ b/octoeverywhere/Proto/HandshakeSyn.py @@ -162,8 +162,15 @@ def OsType(self): return self._tab.Get(octoflatbuffers.number_types.Int8Flags, o + self._tab.Pos) return 0 + # HandshakeSyn + def ReceiveCompressionType(self): + o = octoflatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(36)) + if o != 0: + return self._tab.Get(octoflatbuffers.number_types.Int8Flags, o + self._tab.Pos) + return 2 + def HandshakeSynStart(builder: octoflatbuffers.Builder): - builder.StartObject(16) + builder.StartObject(17) def Start(builder: octoflatbuffers.Builder): HandshakeSynStart(builder) @@ -270,6 +277,12 @@ def HandshakeSynAddOsType(builder: octoflatbuffers.Builder, osType: int): def AddOsType(builder: octoflatbuffers.Builder, osType: int): HandshakeSynAddOsType(builder, osType) +def HandshakeSynAddReceiveCompressionType(builder: octoflatbuffers.Builder, receiveCompressionType: int): + builder.PrependInt8Slot(16, receiveCompressionType, 2) + +def AddReceiveCompressionType(builder: octoflatbuffers.Builder, receiveCompressionType: int): + HandshakeSynAddReceiveCompressionType(builder, receiveCompressionType) + def HandshakeSynEnd(builder: octoflatbuffers.Builder) -> int: return builder.EndObject() diff --git a/octoeverywhere/WebStream/octowebstream.py b/octoeverywhere/WebStream/octowebstream.py index 8bfceb1..aba020b 100644 --- a/octoeverywhere/WebStream/octowebstream.py +++ b/octoeverywhere/WebStream/octowebstream.py @@ -24,7 +24,7 @@ def __init__(self, group=None, target=None, name=None, args=(), kwargs=None, ver self.Logger = args[0] self.Id = args[1] self.OctoSession = args[2] - self.OpenWebStreamMsg = None + self.OpenWebStreamMsg:WebStreamMsg.WebStreamMsg = None self.IsClosed = False self.HasSentCloseMessage = False self.StateLock = threading.Lock() @@ -46,7 +46,7 @@ def __init__(self, group=None, target=None, name=None, args=(), kwargs=None, ver # # This function is called on the main OctoSocket receive thread, so it should pass the # message off to the thread as quickly as possible. - def OnIncomingServerMessage(self, webStreamMsg): + def OnIncomingServerMessage(self, webStreamMsg:WebStreamMsg.WebStreamMsg): # Don't accept messages after we are closed. if self.IsClosed: self.Logger.info("Web stream class "+str(self.Id)+" got a incoming message after it has been closed.") @@ -139,7 +139,7 @@ def mainThread(self): # Timeout after 60 seconds just to check that we aren't closed. # It's important to set this value to None, otherwise on loops it will hold it's old value # which can accidentally re-process old messages. - webStreamMsg = None + webStreamMsg:WebStreamMsg.WebStreamMsg = None try: webStreamMsg = self.MsgQueue.get(timeout=60) except Exception as _: @@ -190,7 +190,7 @@ def mainThread(self): return - def initFromOpenMessage(self, webStreamMsg): + def initFromOpenMessage(self, webStreamMsg:WebStreamMsg.WebStreamMsg): # Sanity check. if self.OpenWebStreamMsg is not None: # Throw so we reset the connection. diff --git a/octoeverywhere/WebStream/octowebstreamhttphelper.py b/octoeverywhere/WebStream/octowebstreamhttphelper.py index 8a5f4cf..c97fe95 100644 --- a/octoeverywhere/WebStream/octowebstreamhttphelper.py +++ b/octoeverywhere/WebStream/octowebstreamhttphelper.py @@ -1,7 +1,6 @@ # namespace: WebStream import time -import zlib import logging import requests @@ -13,6 +12,7 @@ from ..octostreammsgbuilder import OctoStreamMsgBuilder from ..Webcam.webcamhelper import WebcamHelper from ..commandhandler import CommandHandler +from ..compression import Compression, CompressionContext from ..sentry import Sentry from ..compat import Compat from ..Proto import HttpHeader @@ -40,10 +40,12 @@ def __init__(self, streamId, logger:logging.Logger, webStream, webStreamOpenMsg, self.WebStreamOpenMsg = webStreamOpenMsg self.IsClosed = False self.OpenedTime = openedTime + self.CompressionContext = CompressionContext(self.Logger) # Vars for response reading self.BodyReadTempBuffer = None self.ChunkedBodyHasNoContentLengthHeaders = False + self.CompressionType:DataCompression.DataCompression = None self.CompressionTimeSec = -1 self.MissingBoundaryWarningCounter = 0 self.IsUsingFullBodyBuffer = False @@ -88,7 +90,7 @@ def Close(self): # Called when a new message has arrived for this stream from the server. # This function should throw on critical errors, that will reset the connection. # Returning true will case the websocket to close on return. - def IncomingServerMessage(self, webStreamMsg): + def IncomingServerMessage(self, webStreamMsg:WebStreamMsg.WebStreamMsg): # Note this is called on a single thread and will always handle messages # in order as they were sent. @@ -105,9 +107,10 @@ def IncomingServerMessage(self, webStreamMsg): # If we didn't know the upload size, we need to finalize it now self.finalizeUnknownUploadSizeIfNeeded() - # Do the request. This will block this thread until it's done and the - # entire response is sent. - self.executeHttpRequest() + # Do the request. This will block this thread until it's done and the entire response is sent. + # We want to make sure we destroy the compression context after this returns, no matter what. + with self.CompressionContext: + self.executeHttpRequest() # Return true since this stream is now done return True @@ -217,7 +220,6 @@ def executeHttpRequest(self): # This function will check if we want to do a 304 return and update the request correctly. self.checkForNotModifiedCacheAndUpdateResponseIfSo(sendHeaders, octoHttpResult) - # Before we check the headers, check if we are using a full body buffer. # If we are using a full body buffer, we need to ensure the content header is set. This will do a few things: # - It will make the request more efficient since we can allocate the fully know buffer size. @@ -233,7 +235,7 @@ def executeHttpRequest(self): # The FULL buffer size must be set in the content-length, not the compressed size, since the compression is just for our link, it's decompressed when the # message is unpacked. fullContentBufferSize = len(octoHttpResult.FullBodyBuffer) - if octoHttpResult.IsBodyBufferZlibCompressed: + if octoHttpResult.BodyBufferCompressionType != DataCompression.DataCompression.None_: fullContentBufferSize = octoHttpResult.BodyBufferPreCompressSize # See what the current header is (if there is one). If it's set, it should match. @@ -291,6 +293,10 @@ def executeHttpRequest(self): # can. compressBody = self.shouldCompressBody(contentTypeLower, octoHttpResult, contentLength) + # If the content length is known, tell the compression system, which will help performance. + if contentLength is not None: + self.CompressionContext.SetTotalCompressedSizeOfData(contentLength) + # Since streams with unknown content-lengths can run for a while, report now when we start one. # If the status code is 304 or 204, we don't expect content. if self.Logger.isEnabledFor(logging.DEBUG) and contentLength is None and octoHttpResult.StatusCode != 304 and octoHttpResult.StatusCode != 204: @@ -336,6 +342,7 @@ def executeHttpRequest(self): nonCompressedBodyReadSize = 0 lastBodyReadLength = 0 dataOffset = None + compressBody = False else: # Start by reading data from the response. # This function will return a read length of 0 and a null data offset if there's nothing to read. @@ -361,9 +368,9 @@ def executeHttpRequest(self): # Validate. if contentLength is not None and nonCompressedContentReadSizeBytes > contentLength: self.Logger.warn(self.getLogMsgPrefix()+" the http stream read more data than the content length indicated.") - if dataOffset is None and contentLength is not None and nonCompressedContentReadSizeBytes < contentLength: + if dataOffset is not None and contentLength is not None and nonCompressedContentReadSizeBytes < contentLength: # This might happen if the connection closes unexpectedly before the transfer is done. - self.Logger.warn(self.getLogMsgPrefix()+" we expected a fixed length response, but the body read completed before we read it all.") + self.Logger.warn(self.getLogMsgPrefix()+f" we expected a fixed length response, but the body read completed before we read it all. cl:{contentLength}, got:{nonCompressedContentReadSizeBytes} {uri}") # Check if this is the last message. # This is the last message if... @@ -405,7 +412,9 @@ def executeHttpRequest(self): WebStreamMsg.AddFullStreamDataSize(builder, contentLength) if compressBody: # If we are compressing, we need to add what we are using and what the original size was. - WebStreamMsg.AddDataCompression(builder, DataCompression.DataCompression.Zlib) + if self.CompressionType is None: + raise Exception("The body of this message should be compressed but not compression type is set.") + WebStreamMsg.AddDataCompression(builder, self.CompressionType) WebStreamMsg.AddOriginalDataSize(builder, nonCompressedBodyReadSize) if isLastMessage: # If this is the last message because we know the body is all @@ -511,7 +520,7 @@ def finalizeUnknownUploadSizeIfNeeded(self): self.UploadBuffer = self.UploadBuffer[0:self.UploadBytesReceivedSoFar] - def copyUploadDataFromMsg(self, webStreamMsg): + def copyUploadDataFromMsg(self, webStreamMsg:WebStreamMsg.WebStreamMsg): # Check how much data this message has in it. # This size is the size of the full buffer, which is decompressed size if the data is compressed. thisMessageDataLen = webStreamMsg.DataLength() @@ -578,54 +587,76 @@ def copyUploadDataFromMsg(self, webStreamMsg): # A helper, given a web stream message returns it's data buffer, decompressed if needed. - def decompressBufferIfNeeded(self, webStreamMsg): - if webStreamMsg.DataCompression() == DataCompression.DataCompression.Brotli: - raise Exception("decompressBufferIfNeeded Failed - Brotli decompression not possible.") - elif webStreamMsg.DataCompression() == DataCompression.DataCompression.Zlib: - return zlib.decompress(webStreamMsg.DataAsByteArray()) - else: + def decompressBufferIfNeeded(self, webStreamMsg:WebStreamMsg.WebStreamMsg): + # Get the compression type. + compressionType = webStreamMsg.DataCompression() + if compressionType is DataCompression.DataCompression.None_: return webStreamMsg.DataAsByteArray() + # It's compressed, decompress it. + return Compression.Get().Decompress(self.CompressionContext, webStreamMsg.DataAsByteArray(), webStreamMsg.OriginalDataSize(), webStreamMsg.IsDataTransmissionDone(), compressionType) def checkForNotModifiedCacheAndUpdateResponseIfSo(self, sentHeaders, octoHttpResult:OctoHttpRequest.Result): # Check if the sent headers have any conditional http headers. - etag = None - modifiedDate = None + requestEtag = None + requestModifiedDate = None for key in sentHeaders: keyLower = key.lower() if keyLower == "if-modified-since": - modifiedDate = sentHeaders[key] + requestModifiedDate = sentHeaders[key] if keyLower == "if-none-match": - etag = sentHeaders[key] + requestEtag = sentHeaders[key] + # If the request etag starts with the weak indicator, remove it + if requestEtag.startswith("W/"): + requestEtag = requestEtag[2:] # If there were none found, there's nothing do to. - if etag is None and modifiedDate is None: + if requestEtag is None and requestModifiedDate is None: return # Look through the response headers + responseEtag = None + responseModifiedDate = None headers = octoHttpResult.Headers for key in headers: keyLower = key.lower() - if etag is not None and keyLower == "etag": - # Both have etags, check them. - # If the request etag starts with the weak indicator, remove it - if etag.startswith("W/"): - etag = etag[2:] - # Check for an exact match. - if etag == headers[key]: - self.updateResponseFor304(octoHttpResult) - return - if modifiedDate is not None and keyLower == "last-modified": - # There are actual ways to parse and compare these, - # But for now we will just do exact matches. - if modifiedDate == headers[key]: - self.updateResponseFor304(octoHttpResult) - return + if keyLower == "etag": + responseEtag = headers[key] + if keyLower == "last-modified": + responseModifiedDate = headers[key] + if responseEtag is not None and responseModifiedDate is not None: + break + + # See if there are any matches. + # If we have both values, both must match. + convertTo304 = False + # If we have both, both must match + if requestEtag is not None and requestModifiedDate is not None: + if responseEtag is not None and responseModifiedDate is not None and requestEtag == responseEtag and requestModifiedDate == responseModifiedDate: + convertTo304 = True + # If we only have the date, see if it matches + elif requestModifiedDate is not None: + if responseModifiedDate is not None and requestModifiedDate == responseModifiedDate: + convertTo304 = True + # If we only have the etag, see if it matches + elif requestEtag is not None: + if responseEtag is not None and requestEtag == responseEtag: + convertTo304 = True + + # Check if we have something to do. + if convertTo304 is False: + return + + # Convert the response. + self.updateResponseFor304(octoHttpResult) def updateResponseFor304(self, octoHttpResult:OctoHttpRequest.Result): + self.Logger.info(f"Converting request for {octoHttpResult.Url} {octoHttpResult.StatusCode} to a 304.") # First of all, update the status code. octoHttpResult.StatusCode = 304 + # Next, if this was a cached result or a result that has a full body buffer, we need to clear it. + octoHttpResult.ClearFullBodyBuffer() # Remove any headers we don't want to send. Including some of these seems to trip up some browsers. # However, there are some we must send... # Quote - Note that the server generating a 304 response MUST generate any of the following header fields that would have been sent in a 200 (OK) response to the same request: Cache-Control, Content-Location, Date, ETag, Expires, and Vary. @@ -647,13 +678,18 @@ def getLogMsgPrefix(self): # Based on the content-type header, this determines if we would apply compression or not. # Returns true or false - def shouldCompressBody(self, contentTypeLower, octoHttpResult, contentLengthOpt): + def shouldCompressBody(self, contentTypeLower:str, octoHttpResult:OctoHttpRequest.Result, contentLengthOpt:int): # Compression isn't too expensive in terms of cpu cost but for text, it drastically # cuts the size down (ike a 75% reduction.) So we are quite liberal with our compression. - # From testing, we have found that compressing anything smaller than ~200 bytes has not effect - # thus it's not worth doing (it actually makes it slightly larger) - if contentLengthOpt is not None and contentLengthOpt < 200: + # If there is a full body buffer and and it's already compressed, always return true. + # This ensures the message is flagged correctly for compression and the body reading system + # will also read the flag and skip the compression. + if octoHttpResult.BodyBufferCompressionType != DataCompression.DataCompression.None_: + return True + + # Make sure we have a known length and it's not too small to compress. + if contentLengthOpt is not None and contentLengthOpt < Compression.MinSizeToCompress: return False # If we don't know what this is, we don't want to compress it. @@ -662,21 +698,16 @@ def shouldCompressBody(self, contentTypeLower, octoHttpResult, contentLengthOpt) if contentTypeLower is None: return False - # If there is a full body buffer and and it's already compressed, always return true. - # This ensures the message is flagged correctly for compression and the body reading system - # will also read the flag and skip the compression. - if octoHttpResult.IsBodyBufferZlibCompressed: - return True - # We will compress... # - Any thing that has text/ in it # - Anything that says it's javascript # - Anything that says it's json # - Anything that's xml # - Anything that's svg + # - Anything that's a application/octet-stream - moonraker sends unknown file types as these. return (contentTypeLower.find("text/") != -1 or contentTypeLower.find("javascript") != -1 or contentTypeLower.find("json") != -1 or contentTypeLower.find("xml") != -1 - or contentTypeLower.find("svg") != -1) + or contentTypeLower.find("svg") != -1 or contentTypeLower.find("application/octet-stream") != -1) # Reads data from the response body, puts it in a data vector, and returns the offset. @@ -686,15 +717,26 @@ def readContentFromBodyAndMakeDataVector(self, builder, octoHttpResult:OctoHttpR # This is the max size each body read will be. Since we are making local calls, most of the time we will always get this full amount as long as theres more body to read. # This size is a little under the max read buffer on the server, allowing the server to handle the buffers with no copies. # - # 3/24/24 - We did a lot of direct download testing to tweak this buffer size and the server read size, these were the best values able to hit about 223mpbs. + # 3/24/24 - We did a lot of direct download testing to tweak this buffer size and the server read size, these were the best values able to hit about 223mbps. # With the current values, the majority of the time is spent sending the data on the websocket. + # + # But NOTE! This size is the actual size that will be allocated for the read buffer (in the stream class) and then the buffer is sliced by how much + # is read. So we can't make this value too large, or we will be allocating big buffers. + # This is 490kb defaultBodyReadSizeBytes = 490 * 1024 # If we are going to compress this read, use a much higher number. Since most of what we compress is text, # and that text usually compresses down to 25% of the og size, we will use a x4 multiplier. + # We do want to make sure this value isn't too big, because we dont want to allocate a huge buffer on low memory systems. if shouldCompress: defaultBodyReadSizeBytes = defaultBodyReadSizeBytes * 4 + # Finally check if we know the content length of the request. If we do, we will set the buffer to be exactly that value. + # This is a lot more efficient, because we only allocate a buffer the exact size we need for the request. + # But we want to limit the max size of the buffer, so we don't allocate a huge buffer for a large request. + if contentLength_NoneIfNotKnown is not None and contentLength_NoneIfNotKnown < defaultBodyReadSizeBytes: + defaultBodyReadSizeBytes = contentLength_NoneIfNotKnown + # Some requests like snapshot requests will already have a fully read body. In this case we use the existing body buffer instead of reading from the body. finalDataBuffer = None bodyReadStartSec = time.time() @@ -737,7 +779,7 @@ def readContentFromBodyAndMakeDataVector(self, builder, octoHttpResult:OctoHttpR defaultBodyReadSizeBytes = contentLength_NoneIfNotKnown else: # Use a 2mb buffer. - defaultBodyReadSizeBytes = 1024 * 1024 * 1024 * 2 + defaultBodyReadSizeBytes = 1024 * 1024 * 2 finalDataBuffer = self.doBodyRead(octoHttpResult, defaultBodyReadSizeBytes) # Keep track of read times. @@ -760,76 +802,36 @@ def readContentFromBodyAndMakeDataVector(self, builder, octoHttpResult:OctoHttpR # If we have the compat handler, give it the buffer before we finalize the size, as it might want to edit the buffer. if Compat.HasWebRequestResponseHandler(): finalDataBuffer = Compat.GetWebRequestResponseHandler().HandleResponse(responseHandlerContext, octoHttpResult, finalDataBuffer) + # Important! If the response handler has edited the buffer, we need to update the content length to match the new size. + # This is safe to do because currently we always read the entire buffer for a responseHandlerContext into one buffer, thus there's only one read, and this is the read. + # The function that calls readContentFromBodyAndMakeDataVector will correct the content header length in the main class, but we must update the encryption context + # otherwise the zstandard lib encryption will fail. + self.CompressionContext.SetTotalCompressedSizeOfData(len(finalDataBuffer)) # If we were asked to compress, do it originalBufferSize = len(finalDataBuffer) # Check to see if this was a full body buffer, if it was already compressed. - if octoHttpResult.IsBodyBufferZlibCompressed: - # If so, use pre compress size it's supplies. - # And skip compression since it's already done. + if octoHttpResult.BodyBufferCompressionType != DataCompression.DataCompression.None_: + # The full body buffer was already compressed and set, so update the other compression values. originalBufferSize = octoHttpResult.BodyBufferPreCompressSize + if self.CompressionType is not None: + raise Exception(f"The BodyBufferCompressionType tried to be set but the compression was already set! It is {self.CompressionType} and now tried to be {octoHttpResult.BodyBufferCompressionType}") + self.CompressionType = octoHttpResult.BodyBufferCompressionType + # Otherwise, check if we should compress elif shouldCompress: - # Some setups can't install brotli since it requires gcc and c++ to compile native code. - # zlib is part of PY so all plugins us it. Right now it's not worth the tradeoff from testing to enable brotli. - # - # After a good amount of testing, we found that a compression level of 3 is a good tradeoff for both. - # For small to medium size files zlib can actually be better. Brotli starts to be much better in terms of speed - # and compression for larger files. But for now given the file sizes we use here, it's not worth it. - # - # Here's a good quick benchmark on a large js file (4mb) - #2021-12-17 22:37:22,258 - octoprint.plugins.octoeverywhere - INFO - zlib level: 0 time:9.43207740784 size: 815175 og:815104 - #2021-12-17 22:37:22,319 - octoprint.plugins.octoeverywhere - INFO - zlib level: 1 time:58.7220191956 size: 273923 og:815104 - #2021-12-17 22:37:22,383 - octoprint.plugins.octoeverywhere - INFO - zlib level: 2 time:61.7210865021 size: 263366 og:815104 - #2021-12-17 22:37:22,453 - octoprint.plugins.octoeverywhere - INFO - zlib level: 3 time:69.3519115448 size: 256257 og:815104 - #2021-12-17 22:37:22,537 - octoprint.plugins.octoeverywhere - INFO - zlib level: 4 time:81.6609859467 size: 239928 og:815104 - #2021-12-17 22:37:22,650 - octoprint.plugins.octoeverywhere - INFO - zlib level: 5 time:110.955953598 size: 231844 og:815104 - #2021-12-17 22:37:22,803 - octoprint.plugins.octoeverywhere - INFO - zlib level: 6 time:150.192022324 size: 229684 og:815104 - #2021-12-17 22:37:22,972 - octoprint.plugins.octoeverywhere - INFO - zlib level: 7 time:166.711091995 size: 229118 og:815104 - #2021-12-17 22:37:23,196 - octoprint.plugins.octoeverywhere - INFO - zlib level: 8 time:221.390962601 size: 228784 og:815104 - #2021-12-17 22:37:23,442 - octoprint.plugins.octoeverywhere - INFO - zlib level: 9 time:244.188070297 size: 228737 og:815104 - #2021-12-17 22:37:23,477 - octoprint.plugins.octoeverywhere - INFO - brotli level: 0 time:31.9409370422 size: 280540 og:815104 - #2021-12-17 22:37:23,536 - octoprint.plugins.octoeverywhere - INFO - brotli level: 1 time:56.2720298767 size: 267581 og:815104 - #2021-12-17 22:37:23,611 - octoprint.plugins.octoeverywhere - INFO - brotli level: 2 time:72.9219913483 size: 245109 og:815104 - #2021-12-17 22:37:23,703 - octoprint.plugins.octoeverywhere - INFO - brotli level: 3 time:86.4551067352 size: 241472 og:815104 - #2021-12-17 22:37:23,874 - octoprint.plugins.octoeverywhere - INFO - brotli level: 4 time:169.479846954 size: 235446 og:815104 - #2021-12-17 22:37:24,125 - octoprint.plugins.octoeverywhere - INFO - brotli level: 5 time:248.244047165 size: 219928 og:815104 - #2021-12-17 22:37:24,451 - octoprint.plugins.octoeverywhere - INFO - brotli level: 6 time:321.651935577 size: 217598 og:815104 - #2021-12-17 22:37:24,848 - octoprint.plugins.octoeverywhere - INFO - brotli level: 7 time:395.76292038 size: 216307 og:815104 - #2021-12-17 22:37:25,334 - octoprint.plugins.octoeverywhere - INFO - brotli level: 8 time:483.689785004 size: 215660 og:815104 - #2021-12-17 22:37:25,973 - octoprint.plugins.octoeverywhere - INFO - brotli level: 9 time:637.011051178 size: 214962 og:815104 - #2021-12-17 22:37:30,395 - octoprint.plugins.octoeverywhere - INFO - brotli level: 10 time:4420.00603676 size: 202474 og:815104 - #2021-12-17 22:37:40,826 - octoprint.plugins.octoeverywhere - INFO - brotli level: 11 time:10429.7590256 size: 198538 og:815104 - # Here's a more average size file - #2021-12-17 22:45:06,278 - octoprint.plugins.octoeverywhere - INFO - zlib level: 0 time:1.84893608093 size: 13514 og:13503 - #2021-12-17 22:45:06,291 - octoprint.plugins.octoeverywhere - INFO - zlib level: 1 time:1.37400627136 size: 5647 og:13503 - #2021-12-17 22:45:06,298 - octoprint.plugins.octoeverywhere - INFO - zlib level: 2 time:3.87191772461 size: 5550 og:13503 - #2021-12-17 22:45:06,301 - octoprint.plugins.octoeverywhere - INFO - zlib level: 3 time:1.43599510193 size: 5498 og:13503 - #2021-12-17 22:45:06,304 - octoprint.plugins.octoeverywhere - INFO - zlib level: 4 time:1.70516967773 size: 5306 og:13503 - #2021-12-17 22:45:06,308 - octoprint.plugins.octoeverywhere - INFO - zlib level: 5 time:2.17819213867 size: 5227 og:13503 - #2021-12-17 22:45:06,312 - octoprint.plugins.octoeverywhere - INFO - zlib level: 6 time:2.08187103271 size: 5217 og:13503 - #2021-12-17 22:45:06,316 - octoprint.plugins.octoeverywhere - INFO - zlib level: 7 time:2.29096412659 size: 5218 og:13503 - #2021-12-17 22:45:06,320 - octoprint.plugins.octoeverywhere - INFO - zlib level: 8 time:2.12597846985 size: 5218 og:13503 - #2021-12-17 22:45:06,324 - octoprint.plugins.octoeverywhere - INFO - zlib level: 9 time:2.29811668396 size: 5218 og:13503 - #2021-12-17 22:45:06,327 - octoprint.plugins.octoeverywhere - INFO - brotli level: 0 time:1.26886367798 size: 5877 og:13503 - #2021-12-17 22:45:06,330 - octoprint.plugins.octoeverywhere - INFO - brotli level: 1 time:1.18708610535 size: 5828 og:13503 - #2021-12-17 22:45:06,334 - octoprint.plugins.octoeverywhere - INFO - brotli level: 2 time:1.77407264709 size: 5479 og:13503 - #2021-12-17 22:45:06,339 - octoprint.plugins.octoeverywhere - INFO - brotli level: 3 time:2.63094902039 size: 5418 og:13503 - #2021-12-17 22:45:06,345 - octoprint.plugins.octoeverywhere - INFO - brotli level: 4 time:4.88996505737 size: 5335 og:13503 - #2021-12-17 22:45:06,354 - octoprint.plugins.octoeverywhere - INFO - brotli level: 5 time:6.34503364563 size: 5007 og:13503 - #2021-12-17 22:45:06,364 - octoprint.plugins.octoeverywhere - INFO - brotli level: 6 time:8.3749294281 size: 5003 og:13503 - #2021-12-17 22:45:06,384 - octoprint.plugins.octoeverywhere - INFO - brotli level: 7 time:18.7141895294 size: 4994 og:13503 - #2021-12-17 22:45:06,411 - octoprint.plugins.octoeverywhere - INFO - brotli level: 8 time:25.6741046906 size: 4994 og:13503 - #2021-12-17 22:45:06,447 - octoprint.plugins.octoeverywhere - INFO - brotli level: 9 time:33.3149433136 size: 4989 og:13503 - #2021-12-17 22:45:06,499 - octoprint.plugins.octoeverywhere - INFO - brotli level: 10 time:50.5220890045 size: 4609 og:13503 - #2021-12-17 22:45:06,636 - octoprint.plugins.octoeverywhere - INFO - brotli level: 11 time:135.287046432 size: 4503 og:13503 - - start = time.time() - finalDataBuffer = zlib.compress(finalDataBuffer, 3) - if self.CompressionTimeSec == -1: + compressionResult = Compression.Get().Compress(self.CompressionContext, finalDataBuffer) + finalDataBuffer = compressionResult.Bytes + # Init and update the total compression time if needed. + if self.CompressionTimeSec < 0: self.CompressionTimeSec = 0 - self.CompressionTimeSec += (time.time() - start) + self.CompressionTimeSec += compressionResult.CompressionTimeSec + # Set the compression type, this should only be set once and can't change. + if self.CompressionType is None: + self.CompressionType = compressionResult.CompressionType + elif self.CompressionType != compressionResult.CompressionType: + raise Exception(f"The data compression has changed mid stream! It was {self.CompressionType} and now tried to be {compressionResult.CompressionType}") # We have a data buffer, so write it into the builder and return the offset. return (originalBufferSize, len(finalDataBuffer), builder.CreateByteVector(finalDataBuffer)) @@ -989,7 +991,7 @@ def readStreamChunk(self, octoHttpResult:OctoHttpRequest.Result, boundaryStr): return tempBufferFilledSize - def doBodyRead(self, octoHttpResult:OctoHttpRequest.Result, readSize): + def doBodyRead(self, octoHttpResult:OctoHttpRequest.Result, readSize:int): try: # Ensure there's an actual requests lib Response object to read from response = octoHttpResult.ResponseForBodyRead @@ -1002,6 +1004,10 @@ def doBodyRead(self, octoHttpResult:OctoHttpRequest.Result, readSize): # # Thus in our case, we want to just read the response raw. This is what the chunk logic does under the hood anyways, so this path # is more direct and should be more efficient. + # + # Note, whatever size we pass in will be allocated as a buffer, filled, and then sliced. + # So if we pass in a huge value, we will get a big buffer allocated. + # So if we know the size, we should use it, so that the buffer allocated it the same amount that's returned. data = response.raw.read(readSize) # If we got a data buffer return it. diff --git a/octoeverywhere/WebStream/octowebstreamwshelper.py b/octoeverywhere/WebStream/octowebstreamwshelper.py index 2c9df55..9e78b47 100644 --- a/octoeverywhere/WebStream/octowebstreamwshelper.py +++ b/octoeverywhere/WebStream/octowebstreamwshelper.py @@ -1,19 +1,20 @@ # namespace: WebStream -import threading import time -import zlib +import threading import websocket -from .octoheaderimpl import HeaderHelper +from ..mdns import MDns from ..sentry import Sentry +from ..compat import Compat +from ..websocketimpl import Client +from ..localip import LocalIpHelper +from ..compression import Compression, CompressionContext +from .octoheaderimpl import HeaderHelper from ..octohttprequest import OctoHttpRequest -from ..mdns import MDns from ..octostreammsgbuilder import OctoStreamMsgBuilder -from ..localip import LocalIpHelper -from ..websocketimpl import Client -from ..compat import Compat + from ..Proto import WebStreamMsg from ..Proto import MessageContext from ..Proto import WebSocketDataTypes @@ -42,6 +43,7 @@ def __init__(self, streamId, logger, webStream, webStreamOpenMsg, openedTime): self.FirstWsMessageSentToLocal = False self.ResolvedLocalHostnameUrl = None self.LookingForConnectMsgAttempts = 0 + self.CompressionContext = CompressionContext(self.Logger) # These vars indicate if the actual websocket is opened or closed. # This is different from IsClosed, which is tracking if the webstream closed status. @@ -240,13 +242,19 @@ def Close(self): except Exception as _ : pass + # Ensure the compressor is cleaned up + try: + self.CompressionContext.__exit__(None, None, None) + except Exception as e: + Sentry.Exception("Websocket stream helper failed to clean up the compression context.", e) + # Called when a new message has arrived for this stream from the server. # This function should throw on critical errors, that will reset the connection. # Returning true will case the websocket to close on return. # This function is called on it's own thread from the web stream, so it's ok to block # as long as it gets cleaned up when the socket closes. - def IncomingServerMessage(self, webStreamMsg): + def IncomingServerMessage(self, webStreamMsg:WebStreamMsg.WebStreamMsg): # We can get messages from this web stream before the actual websocket has opened and is ready for messages. # If this happens, when we try to send the message on the socket and we will get an error saying "the socket is closed" (which is incorrect, it's not open yet). @@ -267,10 +275,9 @@ def IncomingServerMessage(self, webStreamMsg): buffer = bytearray(0) # If the message is compressed, decompress it. - if webStreamMsg.DataCompression() == DataCompression.DataCompression.Brotli: - raise Exception("IncomingServerMessage Failed - Brotli decompress not possible.") - elif webStreamMsg.DataCompression() == DataCompression.DataCompression.Zlib: - buffer = zlib.decompress(buffer) + compressionType = webStreamMsg.DataCompression() + if compressionType != DataCompression.DataCompression.None_: + buffer = Compression.Get().Decompress(self.CompressionContext, buffer, webStreamMsg.OriginalDataSize(), False, compressionType) # Get the send type. sendType = 0 @@ -366,15 +373,14 @@ def onWsData(self, ws, buffer:bytes, msgType): Sentry.Exception("Websocket stream helper failed to parse websocket for config hash mod.", ex) - # Some messages are large, so compression helps. - # We also don't consider the message type, since binary messages can very easily be - # text as well, and the cost of compression in terms of CPU is low. - usingCompression = len(buffer) > 200 + # Figure out if we should compress the data. + usingCompression = len(buffer) >= Compression.MinSizeToCompress originalDataSize = 0 + compressionResult = None if usingCompression: - # See notes about the quality and such in the readContentFromBodyAndMakeDataVector. originalDataSize = len(buffer) - buffer = zlib.compress(buffer, 3) + compressionResult = Compression.Get().Compress(self.CompressionContext, buffer) + buffer = compressionResult.Bytes # Send the message along! builder = OctoStreamMsgBuilder.CreateBuffer(len(buffer) + 200) @@ -390,7 +396,7 @@ def onWsData(self, ws, buffer:bytes, msgType): WebStreamMsg.AddIsControlFlagsOnly(builder, False) WebStreamMsg.AddWebsocketDataType(builder, sendType) if usingCompression: - WebStreamMsg.AddDataCompression(builder, DataCompression.DataCompression.Zlib) + WebStreamMsg.AddDataCompression(builder, compressionResult.CompressionType) WebStreamMsg.AddOriginalDataSize(builder, originalDataSize) if dataOffset is not None: WebStreamMsg.AddData(builder, dataOffset) diff --git a/octoeverywhere/commandhandler.py b/octoeverywhere/commandhandler.py index 882c0ec..051c671 100644 --- a/octoeverywhere/commandhandler.py +++ b/octoeverywhere/commandhandler.py @@ -381,7 +381,11 @@ def HandleCommand(self, httpInitialContext, postBody_CanBeNone): }).encode(encoding="utf-8") # Build the full result - return OctoHttpRequest.Result(200, {}, OctoStreamMsgBuilder.BytesToString(httpInitialContext.Path()), False, fullBodyBuffer=resultBytes) + # Make sure to set the content type, so the response can be compressed. + headers = { + "Content-Type": "text/json" + } + return OctoHttpRequest.Result(200, headers, OctoStreamMsgBuilder.BytesToString(httpInitialContext.Path()), False, fullBodyBuffer=resultBytes) # The goal here is to keep as much of the common logic as common as possible. diff --git a/octoeverywhere/compression.py b/octoeverywhere/compression.py new file mode 100644 index 0000000..bf0734f --- /dev/null +++ b/octoeverywhere/compression.py @@ -0,0 +1,493 @@ +import os +import sys +import json +import time +import zlib +import logging +import threading +import subprocess +import multiprocessing + +from .sentry import Sentry +from .zstandarddictionary import ZStandardDictionary + +from .Proto.DataCompression import DataCompression + + +# A return type for the compression operation. +class CompressionResult: + def __init__(self, b: bytes, duration:float, compressionType: DataCompression) -> None: + self.Bytes = b + self.CompressionType = compressionType + self.CompressionTimeSec = duration + + +# The compression context should match the lifespan of the compression operation for a set of data. +# For example, one websocket should use the same compression context, so it uses one compression stream. +# This class is not thread safe PER OPERATION so it must only be used by one thread per operation. +# So only one thread can be doing compression, but another thread can be doing decompression. +# This class rents shared resources, so it should be used in with the `with` statement in PY to make sure it's cleaned up. +class CompressionContext: + + # This is the default value used by the zstandard to indicate the full size of the data is unknown. + TOTAL_SIZE_UNKNOWN = -1 + + + def __init__(self, logger:logging.Logger) -> None: + self.Logger = logger + self.ResourceLock = threading.Lock() + self.IsClosed = False + + # Compression - can't be shared to be thread safe + self.Compressor = None + self.StreamWriter = None + self.CompressionByteBuffer:bytes = None + # The compression is more efficient if we know the size of the data of the og data. + self.CompressionTotalSizeOfDataBytes:int = CompressionContext.TOTAL_SIZE_UNKNOWN + + # Decompression - can't be shared to be thread safe + self.Decompressor = None + self.StreamReader = None + self.DecompressionByteBuffer:bytes = None + + + def __del__(self): + # Ensure exit was called before the object is destroyed. + # This ensures we always return the compression contexts + try: + self.__exit__(None, None, None) + except Exception as e: + Sentry.Exception("CompressionContext had an exception on object delete", e) + + + def __enter__(self): + return self + + + def __exit__(self, exc_type, exc_value, traceback): + # Free anything that has been allocated in reverse order. + # We use a lock to ensure we don't leak any of the resources, especially the rented ones. + streamWriter = None + compressor = None + streamReader = None + decompressor = None + with self.ResourceLock: + self.IsClosed = True + + streamWriter = self.StreamWriter + compressor = self.Compressor + self.StreamWriter = None + self.Compressor = None + self.CompressionByteBuffer = None + + streamReader = self.StreamReader + decompressor = self.Decompressor + self.StreamReader = None + self.Decompressor = None + self.DecompressionByteBuffer = None + + # Exit them outside of the lock + if streamWriter is not None: + streamWriter.__exit__(exc_type, exc_value, traceback) + if compressor is not None: + Compression.Get().ReturnZStandardCompressor(compressor) + if streamReader is not None: + streamReader.__exit__(exc_type, exc_value, traceback) + if decompressor is not None: + Compression.Get().ReturnZStandardDecompressor(decompressor) + + + # Ideally, we want to tell the system how much data is being compressed in total. + def SetTotalCompressedSizeOfData(self, totalSizeBytes:int): + if self.StreamWriter is not None: + raise Exception("CompressionContext SetTotalSizeOfData tried to be set after compression started") + self.CompressionTotalSizeOfDataBytes = totalSizeBytes + + + # This is the callback from stream_writer that get called when it has data to write. + def write(self, data): + if self.CompressionByteBuffer is None: + self.CompressionByteBuffer = data + else: + self.CompressionByteBuffer += data + + + # Compresses the data. + # Returns a successful CompressionResult or throws + def Compress(self, data:bytes) -> CompressionResult: + # Ensure we are setup. + startSec = time.time() + with self.ResourceLock: + if self.IsClosed: + raise Exception("The compression context is closed, we can't compress data") + if self.Compressor is None: + self.Compressor = Compression.Get().RentZStandardCompressor() + if self.Compressor is None: + raise Exception("CompressionContext failed to rent a compressor") + + # After a lot of testing, we found that the streaming compression about 80% slower, but that's only 0.1ms in most cases. + # But if it's an actual stream AND WE ARE DOING MULTIPLE COMPRESSES, it can compress UP TO 300% TIMES BETTER, for example with websocket messages. + # If we are only doing one (big) compress, then there's no big compression gain, so we only take a time hit. + # + # Thus, as a good middle ground, if the buffer input is the exact size as we know the full length is, we do a one time compress. + if self.CompressionTotalSizeOfDataBytes == len(data): + return CompressionResult(self.Compressor.compress(data), time.time() - startSec, DataCompression.ZStandard) + + # If the data is size is unknown or this buffer is smaller than it, it's most likely a stream, so the streaming setup works much better. + # Since we are passing the size if known, we can't call flush(zstd.FLUSH_FRAME), since the size indicates the expected full frame size. + with self.ResourceLock: + if self.IsClosed: + raise Exception("The compression context is closed, we can't start a stream writer") + if self.StreamWriter is None: + self.StreamWriter = self.Compressor.stream_writer(self, size=self.CompressionTotalSizeOfDataBytes) + + # Compress this chunk. + self.StreamWriter.write(data) + + # We call flush to get the output that can be independently decompressed, but we don't use the + # zstd.FLUSH_FRAME flag. If we used the zstd.FLUSH_FRAME, we would have to make sure the entire length is written. + self.StreamWriter.flush() + + # Capture the buffer of the written data. + if self.CompressionByteBuffer is None: + raise Exception("CompressionContext failed to get a buffer of the compressed data") + resultBuffer = self.CompressionByteBuffer + self.CompressionByteBuffer = None + + # Done + return CompressionResult(resultBuffer, time.time() - startSec, DataCompression.ZStandard) + + + # This is the callback from stream_reader that get called when it needs more data to read. + def read(self, readSizeBytes:int) -> bytes: + if self.DecompressionByteBuffer is None: + # This is bad. If we return bytes(), which is what is normally done when the stream has ended, it will prevent + # the stream_reader from ever reading again. In our case, we should never hit this, because we don't know how much + # more of the stream there is to read. + # We prevent this from happening by calling read with exactly the uncompressed size of the data. This means that the read + # loop will consume the full buffer, but then never come back for more because it's output all it should have. + raise Exception("CompressionContext read ran out of buffer to read so the stream will be terminated early.") + #return bytes() + + # If the read size is the same as the buffer, we will consume it all at once. + if readSizeBytes >= len(self.DecompressionByteBuffer): + ret = self.DecompressionByteBuffer + self.DecompressionByteBuffer = None + return ret + + # Otherwise, we will consume the exact amount we are asked for. + ret = self.DecompressionByteBuffer[:readSizeBytes] + self.DecompressionByteBuffer = self.DecompressionByteBuffer[readSizeBytes:] + return ret + + + # Given a byte buffer, decompresses the stream and returns the bytes. + def Decompress(self, data:bytes, thisMsgUncompressedDataSize:int, isLastMessage:bool) -> bytes: + # Ensure we are setup. + isFirstMessage = False + with self.ResourceLock: + if self.IsClosed: + raise Exception("The compression context is closed, we can't decompress data") + if self.Decompressor is None: + isFirstMessage = True + self.Decompressor = Compression.Get().RentZStandardDecompressor() + if self.Decompressor is None: + raise Exception("CompressionContext failed to rent a decompressor") + + # Same the the compressor, if this is the first and only message, we use the one time decompress. + # This is faster because for some reason using the stream version of the API for just one message is slower. + if isFirstMessage and isLastMessage: + return self.Decompressor.decompress(data) + + # If the data is size is unknown or this buffer is smaller than it, it's most likely a stream, so the streaming setup works much better. + # Since we are passing the size if known, we can't call flush(zstd.FLUSH_FRAME), since the size indicates the expected full frame size. + with self.ResourceLock: + if self.IsClosed: + raise Exception("The compression context is closed, we can't start a stream reader") + if self.StreamReader is None: + self.StreamReader = self.Decompressor.stream_reader(self) + + # Set the buffer for the decompressor to be read by the read() function + self.DecompressionByteBuffer = data + + # NOTE! It's important to read exactly the amount we are expecting and nothing more. + # The reason is explained in the read() function + return self.StreamReader.read(thisMsgUncompressedDataSize) + + +# A helper class to handle compression for streams. +class Compression: + + # Defines the min size a buffer must be before we compress it. + # There's some small size that's not worth the time to compress, and also compressing it usually makes it bigger. + # That said, zstandard actually does quite well with small payloads, so we can set this quite low. + MinSizeToCompress = 200 + + # Since zstandard can't be a required dep since it will fail on some platforms, we try to install it via the runtime or + # the linux installer if possible. Due to that, this is the package version string they will use ty to to install it. + # We currently have this set to 21, which still supports PY3.7, which is from 2019. + # THIS MUST STAY IN SYNC WITH THE VERSION IN THE Dockerfile and the GitHub actions linter file. + ZStandardPipPackageString = "zstandard>=0.21.0,<0.23.0" + ZStandardMinCoreCountForInstall = 3 + + _Instance = None + + @staticmethod + def Init(logger: logging.Logger, localFileStoragePath:str): + Compression._Instance = Compression(logger, localFileStoragePath) + + + @staticmethod + def Get(): + return Compression._Instance + + + def __init__(self, logger: logging.Logger, localFileStoragePath:str) -> None: + self.Logger = logger + self.LocalFileStoragePath = localFileStoragePath + self.ZStandardCompressorPool = [] + self.ZStandardCompressorPoolLock = threading.Lock() + self.ZStandardCompressorCreatedCount = 0 + + self.ZStandardDecompressorPool = [] + self.ZStandardDecompressorPoolLock = threading.Lock() + self.ZStandardDecompressorCreatedCount = 0 + + # Determine the thread count we will allow zstandard to use. + # If there are 3 or less cores, we will only use one thread. + # If there are 4 or more cores, we will use all but 2. + self.ZStandardThreadCount = 1 + cpuCores = multiprocessing.cpu_count() + if cpuCores <= 3: + self.ZStandardThreadCount = 1 + else: + self.ZStandardThreadCount = cpuCores - 2 + + # Always init the zstandard singleton, even if we aren't using zstandard. + ZStandardDictionary.Init(logger) + + # Try to load the zstandard library, if it fails, we won't use it. + # Some systems don't have the native lib this will try to load, so we will fall back to zlib. + self.CanUseZStandardLib = False + try: + #pylint: disable=import-outside-toplevel,unused-import + import zstandard as zstd + + # Since we are using zlib, try to load the pre-trained dictionary. + # This will throw if it fails, and we must load this dict to use zstandard, because the server expects it. + ZStandardDictionary.Get().InitPreComputedDict() + + # Only set this flag after everything is setup and good. + self.CanUseZStandardLib = True + self.Logger.info(f"Compression is using zstandard with {self.ZStandardThreadCount} threads") + + # Once the state is set, make a few compressors and decompressors so they are cached and ready to go. + c = self.RentZStandardCompressor() + c2 = self.RentZStandardCompressor() + self.ReturnZStandardCompressor(c) + self.ReturnZStandardCompressor(c2) + + d = self.RentZStandardDecompressor() + d2 = self.RentZStandardDecompressor() + self.ReturnZStandardDecompressor(d) + self.ReturnZStandardDecompressor(d2) + except Exception as e: + self.Logger.info(f"Failed to load the zstandard lib, so we won't use it. Error: {e}") + + # If we can't use zstandard, we assume it's not installed since it doesn't install as a required dependency. + # In that case, we will use this function to try to install it async, and it will be used on the next restart. + # But, if the system has two or less cores, dont try to install, because it's probably not powerful enough to use it. + if self.CanUseZStandardLib is False and cpuCores >= Compression.ZStandardMinCoreCountForInstall: + self._TryInstallZStandardIfNeededAsync() + + + # Given a buffer of data, compress it using the best available compression library. + def Compress(self, compressionContext:CompressionContext, data: bytes) -> CompressionResult: + # If we have zstandard lib, use that, since it's better. + if self.CanUseZStandardLib: + # If we are training, submit the data to be sampled. + # ZStandardDictionary.Get().SubmitData(data) + return compressionContext.Compress(data) + + # If we can't use zStandard lib, fallback to zlib + startSec = time.time() + compressed = zlib.compress(data, 3) + return CompressionResult(compressed, time.time() - startSec, DataCompression.Zlib) + + + # Given a buffer of data and the compression type, decompresses it. + def Decompress(self, compressionContext:CompressionContext, data:bytes, thisMsgUncompressedDataSize:int, isLastMessage:bool, compressionType: DataCompression) -> bytes: + # Decompress depending on what type of compression was used. + if compressionType == DataCompression.Zlib: + return zlib.decompress(data) + elif compressionType == DataCompression.ZStandard: + if self.CanUseZStandardLib is False: + raise Exception("We tried to decompress data using DataCompression.ZStandard, but we can't support that library on this system.") + return compressionContext.Decompress(data, thisMsgUncompressedDataSize, isLastMessage) + # This is logic we use if we want to train the zstandard lib. + # data = compressionContext.Decompress(data, thisMsgUncompressedDataSize, isLastMessage) + # ZStandardDictionary.Get().SubmitData(data) + # return data + else: + raise Exception(f"Unknown compression type: {compressionType}") + + + # Returns a compressor or None if it fails to load. + # The compressor warps the zstandard lib context, they are reusable but not thread safe. + def RentZStandardCompressor(self): + if self.CanUseZStandardLib is False: + return None + try: + with self.ZStandardCompressorPoolLock: + if len(self.ZStandardCompressorPool) > 0: + return self.ZStandardCompressorPool.pop() + + # Report how many we have created for leak detection. + self.ZStandardCompressorCreatedCount += 1 + if self.ZStandardCompressorCreatedCount > 40: + self.Logger.warn(f"Compression zstandard compressor pool has created {self.ZStandardCompressorCreatedCount} items, there might be a leak") + + #pylint: disable=import-outside-toplevel + import zstandard as zstd + # We must use the pre-trained dict, since the service uses it as well and it must match. + return zstd.ZstdCompressor(threads=self.ZStandardThreadCount, dict_data=ZStandardDictionary.Get().PreTrainedDict) + except Exception as e: + self.Logger.error(f"Failed to rent zstandard compressor. Error: {e}") + return None + + + # Puts the compressor back into the pool + def ReturnZStandardCompressor(self, compressor): + if compressor is None: + return + with self.ZStandardCompressorPoolLock: + self.ZStandardCompressorPool.append(compressor) + + + # Returns a decompressor or None if it fails to load. + # The decompressor warps the zstandard lib context, they are reusable but not thread safe. + def RentZStandardDecompressor(self): + if self.CanUseZStandardLib is False: + return None + try: + with self.ZStandardDecompressorPoolLock: + if len(self.ZStandardDecompressorPool) > 0: + return self.ZStandardDecompressorPool.pop() + + # Report how many we have created for leak detection. + self.ZStandardDecompressorCreatedCount += 1 + if self.ZStandardDecompressorCreatedCount > 40: + self.Logger.warn(f"Compression zstandard decompressor pool has created {self.ZStandardDecompressorCreatedCount} items, there might be a leak") + + #pylint: disable=import-outside-toplevel + import zstandard as zstd + # We must use the pre-trained dict, since the service uses it as well and it must match. + return zstd.ZstdDecompressor(dict_data=ZStandardDictionary.Get().PreTrainedDict) + except Exception as e: + self.Logger.error(f"Failed to rent zstandard decompressor. Error: {e}") + return None + + + # Puts the decompressor back into the pool + def ReturnZStandardDecompressor(self, decompressor): + if decompressor is None: + return + with self.ZStandardDecompressorPoolLock: + self.ZStandardDecompressorPool.append(decompressor) + + + # If we can't use zstandard, we assume it's not installed since it doesn't install as a required dependency. + # In that case, we will use this function to try to install it async, and it will be used on the next restart. + def _TryInstallZStandardIfNeededAsync(self): + threading.Thread(target=self._TryInstallZStandardIfNeeded, daemon=True).start() + + + def _TryInstallZStandardIfNeeded(self): + lastAttemptFileName = "CompressionData.json" + try: + # First, see if we need to try to do this again. + filePath = os.path.join(self.LocalFileStoragePath, lastAttemptFileName) + if os.path.exists(filePath): + with open(filePath, encoding="utf-8") as f: + data = json.load(f) + if "LastUpdateTimeSec" in data: + lastUpdateTimeSec = float(data["LastUpdateTimeSec"]) + # If the most recent attempt was less than 30 days ago, we won't try again. + if time.time() - lastUpdateTimeSec < 30 * 24 * 60 * 60: + return + + # We are going to update, write a file now with the current time. + with open(filePath, encoding="utf-8", mode="w") as f: + data = { + "LastUpdateTimeSec": time.time() + } + json.dump(data, f) + + # Try to do the update now. + # Limit the install, but give it a longer timeout since it might try to compile. + # Use `sys.executable` to make sure we get our virtual env python. + result = subprocess.run([sys.executable, '-m', 'pip', 'install', Compression.ZStandardPipPackageString], timeout=60.0, check=False, capture_output=True) + if result.returncode == 0: + self.Logger.info(f"Pip install/update of {sys.executable} {Compression.ZStandardPipPackageString} successful.") + return + self.Logger.info(f"Compression pip install failed. {sys.executable} {Compression.ZStandardPipPackageString}. stdout:{result.stdout} - stderr:{result.stderr}") + except Exception as e: + self.Logger.error(f"Compression failed to pip install zstandard lib. {e}") + +# +# This is an old comment, from before zstandard lib. But it still has useful info about zlib and brotli +# For zstandard, we found that it's faster and compresses way better, especially on small messages if it can stream like the websocket. +# +# Some setups can't install brotli since it requires gcc and c++ to compile native code. +# zlib is part of PY so all plugins us it. Right now it's not worth the tradeoff from testing to enable brotli. +# +# After a good amount of testing, we found that a compression level of 3 is a good tradeoff for both. +# For small to medium size files zlib can actually be better. Brotli starts to be much better in terms of speed +# and compression for larger files. But for now given the file sizes we use here, it's not worth it. +# +# Here's a good quick benchmark on a large js file (4mb) +#2021-12-17 22:37:22,258 - octoprint.plugins.octoeverywhere - INFO - zlib level: 0 time:9.43207740784 size: 815175 og:815104 +#2021-12-17 22:37:22,319 - octoprint.plugins.octoeverywhere - INFO - zlib level: 1 time:58.7220191956 size: 273923 og:815104 +#2021-12-17 22:37:22,383 - octoprint.plugins.octoeverywhere - INFO - zlib level: 2 time:61.7210865021 size: 263366 og:815104 +#2021-12-17 22:37:22,453 - octoprint.plugins.octoeverywhere - INFO - zlib level: 3 time:69.3519115448 size: 256257 og:815104 +#2021-12-17 22:37:22,537 - octoprint.plugins.octoeverywhere - INFO - zlib level: 4 time:81.6609859467 size: 239928 og:815104 +#2021-12-17 22:37:22,650 - octoprint.plugins.octoeverywhere - INFO - zlib level: 5 time:110.955953598 size: 231844 og:815104 +#2021-12-17 22:37:22,803 - octoprint.plugins.octoeverywhere - INFO - zlib level: 6 time:150.192022324 size: 229684 og:815104 +#2021-12-17 22:37:22,972 - octoprint.plugins.octoeverywhere - INFO - zlib level: 7 time:166.711091995 size: 229118 og:815104 +#2021-12-17 22:37:23,196 - octoprint.plugins.octoeverywhere - INFO - zlib level: 8 time:221.390962601 size: 228784 og:815104 +#2021-12-17 22:37:23,442 - octoprint.plugins.octoeverywhere - INFO - zlib level: 9 time:244.188070297 size: 228737 og:815104 +#2021-12-17 22:37:23,477 - octoprint.plugins.octoeverywhere - INFO - brotli level: 0 time:31.9409370422 size: 280540 og:815104 +#2021-12-17 22:37:23,536 - octoprint.plugins.octoeverywhere - INFO - brotli level: 1 time:56.2720298767 size: 267581 og:815104 +#2021-12-17 22:37:23,611 - octoprint.plugins.octoeverywhere - INFO - brotli level: 2 time:72.9219913483 size: 245109 og:815104 +#2021-12-17 22:37:23,703 - octoprint.plugins.octoeverywhere - INFO - brotli level: 3 time:86.4551067352 size: 241472 og:815104 +#2021-12-17 22:37:23,874 - octoprint.plugins.octoeverywhere - INFO - brotli level: 4 time:169.479846954 size: 235446 og:815104 +#2021-12-17 22:37:24,125 - octoprint.plugins.octoeverywhere - INFO - brotli level: 5 time:248.244047165 size: 219928 og:815104 +#2021-12-17 22:37:24,451 - octoprint.plugins.octoeverywhere - INFO - brotli level: 6 time:321.651935577 size: 217598 og:815104 +#2021-12-17 22:37:24,848 - octoprint.plugins.octoeverywhere - INFO - brotli level: 7 time:395.76292038 size: 216307 og:815104 +#2021-12-17 22:37:25,334 - octoprint.plugins.octoeverywhere - INFO - brotli level: 8 time:483.689785004 size: 215660 og:815104 +#2021-12-17 22:37:25,973 - octoprint.plugins.octoeverywhere - INFO - brotli level: 9 time:637.011051178 size: 214962 og:815104 +#2021-12-17 22:37:30,395 - octoprint.plugins.octoeverywhere - INFO - brotli level: 10 time:4420.00603676 size: 202474 og:815104 +#2021-12-17 22:37:40,826 - octoprint.plugins.octoeverywhere - INFO - brotli level: 11 time:10429.7590256 size: 198538 og:815104 +# Here's a more average size file +#2021-12-17 22:45:06,278 - octoprint.plugins.octoeverywhere - INFO - zlib level: 0 time:1.84893608093 size: 13514 og:13503 +#2021-12-17 22:45:06,291 - octoprint.plugins.octoeverywhere - INFO - zlib level: 1 time:1.37400627136 size: 5647 og:13503 +#2021-12-17 22:45:06,298 - octoprint.plugins.octoeverywhere - INFO - zlib level: 2 time:3.87191772461 size: 5550 og:13503 +#2021-12-17 22:45:06,301 - octoprint.plugins.octoeverywhere - INFO - zlib level: 3 time:1.43599510193 size: 5498 og:13503 +#2021-12-17 22:45:06,304 - octoprint.plugins.octoeverywhere - INFO - zlib level: 4 time:1.70516967773 size: 5306 og:13503 +#2021-12-17 22:45:06,308 - octoprint.plugins.octoeverywhere - INFO - zlib level: 5 time:2.17819213867 size: 5227 og:13503 +#2021-12-17 22:45:06,312 - octoprint.plugins.octoeverywhere - INFO - zlib level: 6 time:2.08187103271 size: 5217 og:13503 +#2021-12-17 22:45:06,316 - octoprint.plugins.octoeverywhere - INFO - zlib level: 7 time:2.29096412659 size: 5218 og:13503 +#2021-12-17 22:45:06,320 - octoprint.plugins.octoeverywhere - INFO - zlib level: 8 time:2.12597846985 size: 5218 og:13503 +#2021-12-17 22:45:06,324 - octoprint.plugins.octoeverywhere - INFO - zlib level: 9 time:2.29811668396 size: 5218 og:13503 +#2021-12-17 22:45:06,327 - octoprint.plugins.octoeverywhere - INFO - brotli level: 0 time:1.26886367798 size: 5877 og:13503 +#2021-12-17 22:45:06,330 - octoprint.plugins.octoeverywhere - INFO - brotli level: 1 time:1.18708610535 size: 5828 og:13503 +#2021-12-17 22:45:06,334 - octoprint.plugins.octoeverywhere - INFO - brotli level: 2 time:1.77407264709 size: 5479 og:13503 +#2021-12-17 22:45:06,339 - octoprint.plugins.octoeverywhere - INFO - brotli level: 3 time:2.63094902039 size: 5418 og:13503 +#2021-12-17 22:45:06,345 - octoprint.plugins.octoeverywhere - INFO - brotli level: 4 time:4.88996505737 size: 5335 og:13503 +#2021-12-17 22:45:06,354 - octoprint.plugins.octoeverywhere - INFO - brotli level: 5 time:6.34503364563 size: 5007 og:13503 +#2021-12-17 22:45:06,364 - octoprint.plugins.octoeverywhere - INFO - brotli level: 6 time:8.3749294281 size: 5003 og:13503 +#2021-12-17 22:45:06,384 - octoprint.plugins.octoeverywhere - INFO - brotli level: 7 time:18.7141895294 size: 4994 og:13503 +#2021-12-17 22:45:06,411 - octoprint.plugins.octoeverywhere - INFO - brotli level: 8 time:25.6741046906 size: 4994 og:13503 +#2021-12-17 22:45:06,447 - octoprint.plugins.octoeverywhere - INFO - brotli level: 9 time:33.3149433136 size: 4989 og:13503 +#2021-12-17 22:45:06,499 - octoprint.plugins.octoeverywhere - INFO - brotli level: 10 time:50.5220890045 size: 4609 og:13503 +#2021-12-17 22:45:06,636 - octoprint.plugins.octoeverywhere - INFO - brotli level: 11 time:135.287046432 size: 4503 og:13503 diff --git a/octoeverywhere/octohttprequest.py b/octoeverywhere/octohttprequest.py index a56bba5..deb1633 100644 --- a/octoeverywhere/octohttprequest.py +++ b/octoeverywhere/octohttprequest.py @@ -7,7 +7,10 @@ from .octostreammsgbuilder import OctoStreamMsgBuilder from .mdns import MDns from .compat import Compat + from .Proto.PathTypes import PathTypes +from .Proto.DataCompression import DataCompression + class OctoHttpRequest: LocalHttpProxyPort = 80 @@ -85,7 +88,7 @@ def __init__(self, statusCode:int, headers:dict, url:str, didFallback:bool, full self._requestLibResponseObj = requestLibResponseObj self._didFallback:bool = didFallback self._fullBodyBuffer = fullBodyBuffer - self._isZlibCompressed:bool = False + self._bodyCompressionType = DataCompression.None_ self._fullBodyBufferPreCompressedSize:int = 0 self.SetFullBodyBuffer(fullBodyBuffer) self._customBodyStreamCallback = customBodyStreamCallback @@ -117,9 +120,9 @@ def FullBodyBuffer(self) -> bytearray: return self._fullBodyBuffer @property - def IsBodyBufferZlibCompressed(self) -> bool: - # There must be a buffer and the flag must be set. - return self._isZlibCompressed and self._fullBodyBuffer is not None + def BodyBufferCompressionType(self) -> DataCompression: + # Defaults to None + return self._bodyCompressionType @property def BodyBufferPreCompressSize(self) -> int: @@ -130,13 +133,20 @@ def BodyBufferPreCompressSize(self) -> int: # Note the buffer can be bytes or bytearray object! # A bytes object is more efficient, but bytearray can be edited. - def SetFullBodyBuffer(self, buffer, isZlibCompressed:bool = False, preCompressedSize:int = 0): + def SetFullBodyBuffer(self, buffer, compressionType:DataCompression = DataCompression.None_, preCompressedSize:int = 0): self._fullBodyBuffer = buffer - self._isZlibCompressed = isZlibCompressed + self._bodyCompressionType = compressionType self._fullBodyBufferPreCompressedSize = preCompressedSize - if isZlibCompressed and preCompressedSize <= 0: + if compressionType != DataCompression.None_ and preCompressedSize <= 0: raise Exception("The pre-compression full size must be set if the buffer is compressed.") + # It's important we clear all of the vars that are set above. + # This is used by the system that updates the request object with a 304 if the cache headers match. + def ClearFullBodyBuffer(self): + self._fullBodyBuffer = None + self._bodyCompressionType = DataCompression.None_ + self._fullBodyBufferPreCompressedSize = 0 + # Since most things use request Stream=True, this is a helpful util that will read the entire # content of a request and return it. Note if the request has no defined length, this will read # as long as the stream will go. diff --git a/octoeverywhere/octosessionimpl.py b/octoeverywhere/octosessionimpl.py index a799149..5db8880 100644 --- a/octoeverywhere/octosessionimpl.py +++ b/octoeverywhere/octosessionimpl.py @@ -17,6 +17,7 @@ from .sentry import Sentry from .ostypeidentifier import OsTypeIdentifier from .threaddebug import ThreadDebug +from .compression import Compression from .Proto import OctoStreamMessage from .Proto import HandshakeAck @@ -25,6 +26,7 @@ from .Proto import OctoNotification from .Proto import OctoNotificationTypes from .Proto import OctoSummon +from .Proto.DataCompression import DataCompression class OctoSession: @@ -243,10 +245,16 @@ def StartHandshake(self, summonMethod): raise Exception("Rsa challenge generation failed.") rasChallengeKeyVerInt = ServerAuthHelper.c_ServerAuthKeyVersion + # Define which type of compression we can receive (beyond None) + # Ideally this is zstandard lib, but all client must support zlib, so we can fallback to it. + receiveCompressionType = DataCompression.Zlib + if Compression.Get().CanUseZStandardLib: + receiveCompressionType = DataCompression.ZStandard + # Build the message buf = OctoStreamMsgBuilder.BuildHandshakeSyn(self.PrinterId, self.PrivateKey, self.isPrimarySession, self.PluginVersion, OctoHttpRequest.GetLocalHttpProxyPort(), LocalIpHelper.TryToGetLocalIp(), - rasChallenge, rasChallengeKeyVerInt, summonMethod, self.ServerHostType, self.IsCompanion, OsTypeIdentifier.DetectOsType()) + rasChallenge, rasChallengeKeyVerInt, summonMethod, self.ServerHostType, self.IsCompanion, OsTypeIdentifier.DetectOsType(), receiveCompressionType) # Send! self.OctoStream.SendMsg(buf) diff --git a/octoeverywhere/octostreammsgbuilder.py b/octoeverywhere/octostreammsgbuilder.py index 44dea91..521713a 100644 --- a/octoeverywhere/octostreammsgbuilder.py +++ b/octoeverywhere/octostreammsgbuilder.py @@ -4,12 +4,13 @@ from .Proto import HandshakeSyn from .Proto import OctoStreamMessage from .Proto import OsType +from .Proto.DataCompression import DataCompression # A helper class that builds our OctoStream messages as flatbuffers. class OctoStreamMsgBuilder: @staticmethod - def BuildHandshakeSyn(printerId, privateKey, isPrimarySession, pluginVersion, localHttpProxyPort, localIp, rsaChallenge, rasKeyVersionInt, summonMethod, serverHostType, isCompanion, osType:OsType.OsType): + def BuildHandshakeSyn(printerId, privateKey, isPrimarySession, pluginVersion, localHttpProxyPort, localIp, rsaChallenge, rasKeyVersionInt, summonMethod, serverHostType, isCompanion, osType:OsType.OsType, receiveCompressionType:DataCompression): # Get a buffer builder = OctoStreamMsgBuilder.CreateBuffer(500) @@ -39,6 +40,7 @@ def BuildHandshakeSyn(printerId, privateKey, isPrimarySession, pluginVersion, lo HandshakeSyn.AddRsaChallenge(builder, rasChallengeOffset) HandshakeSyn.AddRasChallengeVersion(builder, rasKeyVersionInt) HandshakeSyn.AddOsType(builder, osType) + HandshakeSyn.AddReceiveCompressionType(builder, receiveCompressionType) synOffset = HandshakeSyn.End(builder) return OctoStreamMsgBuilder.CreateOctoStreamMsgAndFinalize(builder, MessageContext.MessageContext.HandshakeSyn, synOffset) diff --git a/octoeverywhere/zstandarddictionary.py b/octoeverywhere/zstandarddictionary.py new file mode 100644 index 0000000..4826cef --- /dev/null +++ b/octoeverywhere/zstandarddictionary.py @@ -0,0 +1,134 @@ +import os +import glob +import random +import base64 +import logging + +# A helper classed used for training the zstandard lib pre made dictionary. +# This is only used for training the dictionary, so it's not used in the main code. +class ZStandardDictionary: + + _Instance = None + + # These are only used for dev building. + _TrainingPath = "/home/pi/zstandard-training-samples" + _OutputDictFilePath = "/home/pi/zstandard-gen-dict-base64.data" + + + @staticmethod + def Init(logger:logging.Logger): + ZStandardDictionary._Instance = ZStandardDictionary(logger) + + + @staticmethod + def Get(): + return ZStandardDictionary._Instance + + + def __init__(self, logger:logging.Logger) -> None: + self.Logger = logger + self.TrainingDataNamePrefix:str = None + + # This will be None if we aren't using zstandard in this runtime. + self.PreTrainedDict = None + + + # The check for zstandard lib must be made before we can call this, but if we are using zstandard, we must load this dict. + def InitPreComputedDict(self): + # To make things easier, we include the dict in the source code as a based64 encoded string. + # This prevents us from doing any kind of file IO or network calls to load the dict. + dictData = base64.b64decode(ZStandardDictionary.c_Dict1) + + # We can input zlib, because this class is only inited when the compression class has already checked for zlib support. + #pylint: disable=import-outside-toplevel,unused-import + import zstandard as zstd + + # Load the dict from the data. + localDict = zstd.ZstdCompressionDict(dictData, dict_type=zstd.DICT_TYPE_FULLDICT) + + # Doing pre-compute now makes it so we don't have to use compute the dict on first use. + # We must specify a level, so we use the same level we use elsewhere, which is the default of 3. + localDict.precompute_compress(level=3) + + # Success! We are using the pre-trained dict, so set it. + self.PreTrainedDict = localDict + self.Logger.info(f"ZStandard Dict Training loaded. Data Length:{len(self.PreTrainedDict.as_bytes())} DictID:{self.PreTrainedDict.dict_id()}") + + + # DEV ONLY + # Used only in dev builds to init training data samples. + # You must also add SubmitData into the Compression class to get the samples submitted. + def InitTrainingOutputDataFile(self, namePrefix:str): + if input(f"Are you sure you want to add to the training data with prefix [{namePrefix}]? (y/n) ") != "y": + return + self.TrainingDataNamePrefix = namePrefix + # Ensure the training path exists. + if not os.path.exists(ZStandardDictionary._TrainingPath): + os.makedirs(ZStandardDictionary._TrainingPath) + + + # DEV ONLY + # This should be called by everything that's compressing data to sample it. + # The training data file should include as much data as we can from all platforms. + # To start training, add this to the Compression.Compress and Compress.Decompress functions if we are using zstandard. + def SubmitData(self, data:bytes) -> None: + # Check state to see if we are training. + if self.TrainingDataNamePrefix is None: + self.Logger.warn("ZStandardDictionary.SubmitData was called but we aren't training!") + return + + try: + # Get a random file name. + fileId = random.random() * 100000000 + fileName = f"{self.TrainingDataNamePrefix}-{fileId}.txt" + self.Logger.info(f"Writing {len(data)} bytes to the training file: {fileName}") + + # Write the data. + with open(os.path.join(ZStandardDictionary._TrainingPath, fileName), "w", encoding="utf-8") as f: + f.write(data.decode("utf-8")) + + except Exception as e: + self.Logger.error(f"ZStandardDictionary failed to write data to the file. Error: {e}") + + + # Used by dev builds to build a new training dict. + def BuildTrainingDict(self): + try: + #pylint: disable=import-outside-toplevel + import zstandard as zstd + + # Train a new dict. + # Collect all of the samples that are in the samples folder. + inputSamples = [] + searchStr = os.path.join(ZStandardDictionary._TrainingPath, "*") + for file in glob.glob(searchStr): + with open(file, "rb") as f: + inputSamples.append(f.read()) + + # Options: + # - The dict size can be whatever we want it to be. + # - The dict_id is a built in id for the dict, we should rev it every time we update the dict. + # - Threads defines how many threads will be used when trying to optimize the function prams. + # - Steps defines how many steps we will take when optimize the function prams. + self.Logger.info(f"ZStandard Dict starting training on {len(inputSamples)} samples.") + dataDict = zstd.train_dictionary(dict_size=112640, samples=inputSamples, dict_id=1, threads=-1, steps=100) + + # Done! + # The k and d values are only used for the training process. + self.Logger.info(f"ZStandard Dict Training done! Data Length:{len(dataDict.as_bytes())} K: {dataDict.k} D: {dataDict.d} DictID: {dataDict.dict_id()}") + + # Base64 encode the output + if input("Do you want to replace the base64 encoded data file on disk? (y/n) ") == "y": + with open(ZStandardDictionary._OutputDictFilePath, "wb") as f: + en = base64.b64encode(dataDict.as_bytes()) + self.Logger.info(f"test {len(en)} {en}") + f.write(en) + + except Exception as e: + self.Logger.error(f"ZStandardDictionary failed to BuildTrainingDict. Error: {e}") + + + # This is the pre-made dict version 1. + # This dict must match what's used in the service, see the service notes for more info. + #pylint: disable=line-too-long + c_Dict1 = "N6Qw7AEAAABAEKAytQ3DMMw5iqIoiqIoiqI4GBpTM9tdNtoufs1Mdi2bdvyO58Xn+mQi/a1JefMkKSkpqVSytFkyPKOPo0ddDoMBAAwQjIbjEZFQLNhn6QMEQNGNSIWPKSOBOJYkOYyiIISQMcYQQgwxxoiMjAwGAAAAtBSri9QpZAgAAAAAAAAAAAAAAAEAAAAEAAAACAAAAGdpbiBmYWlsZWQiKSx0ZXh0OmdldHRleHQoIlVzZXIgdW5rbm93biwgd3JvbmcgcGFzc3dvcmQgb3IgYWNjb3VudCBkZWFjdGl2YXRlZCIpLHR5cGU6ImVycm9yIn0pO2JyZWFrO319fSk7fTt2YXIgX2xvZ291dEluUHJvZ3Jlc3M9ZmFsc2U7c2VsZi5sb2dvdXQ9ZnVuY3Rpb24odGFyZ2V0LGV2ZW50KXtpZighc2VsZi5sb2dvdXRTdXBwb3J0ZWQoKSl7ZXZlbnQuc3RvcFByb3BhZ2F0aW9uKCk7cmV0dXJuO30KaWYoX2xvZ291dEluUHJvZ3Jlc3MpcmV0dXJuO19sb2dvdXRJblByb2dyZXNzPXRydWU7cmV0dXJuIE9jdG9QcmludC5icm93c2VyLmxvZ291dCgpLmRvbmUoZnVuY3Rpb24ocmVzcG9uc2Upe25ldyBQTm90aWZ5KHt0aXRsZTpnZXR0ZXh0KCJMb2dvdXQgc3VjY2Vzc2Z1bCIpLHRleHQ6Z2V0dGV4dCgiWW91IGFyZSBub3cgbG9nZ2VkIG91dCIpLHR5cGU6InN1Y2Nlc3MifSk7c2VsZi5mcm9tUmVzcG9uc2UocmVzcG9uc2UpO30pLmZhaWwoZnVuY3Rpb24oZXJyb3Ipe2lmKGVycm9yJiZlcnJvci5zdGF0dXM9PT00MDMpe3NlbGYuZnJvbVJlc3BvbnNlKGZhbHNlKTt9fSkuYWx3YXlzKGZ1bmN0aW9uKCl7X2xvZ291dEluUHJvZ3Jlc3M9ZmFsc2U7fSk7fTtzZWxmLnByZXBhcmVMb2dpbj1mdW5jdGlvbihkYXRhLGV2ZW50KXtpZihldmVudCYmZXZlbnQucHJldmVudERlZmF1bHQpe2V2ZW50LnByZXZlbnREZWZhdWx0KCk7fQpzZWxmLmxvZ2luKCk7fTtzZWxmLm9uRGF0YVVwZGF0ZXJSZWF1dGhSZXF1aXJlZD1mdW5jdGlvbihyZWFzb24pe2lmKHJlYXNvbj09PSJsb2dvdXQifHxyZWFzb249PT0ic3RhbGUifG94LXNoYWRvdzpub25lfS5idG4tbGlua3tib3JkZXItY29sb3I6dHJhbnNwYXJlbnQ7Y29sb3I6IzA4Yzstd2Via2l0LWJvcmRlci1yYWRpdXM6MDstbW96LWJvcmRlci1yYWRpdXM6MDtib3JkZXItcmFkaXVzOjB9LmJ0bi1saW5rOmZvY3VzLC5idG4tbGluazpob3Zlcntjb2xvcjojMDA1NTgwO3RleHQtZGVjb3JhdGlvbjp1bmRlcmxpbmU7YmFja2dyb3VuZC1jb2xvcjp0cmFuc3BhcmVudH0uYnRuLWxpbmtbZGlzYWJsZWRdOmZvY3VzLC5idG4tbGlua1tkaXNhYmxlZF06aG92ZXJ7Y29sb3I6IzMzMzt0ZXh0LWRlY29yYXRpb246bm9uZX0uY2xlYXJmaXh7Knpvb206MX0uY2xlYXJmaXg6YWZ0ZXIsLmNsZWFyZml4OmJlZm9yZXtkaXNwbGF5OnRhYmxlO2NvbnRlbnQ6IiI7bGluZS1oZWlnaHQ6MH0uY2xlYXJmaXg6YWZ0ZXJ7Y2xlYXI6Ym90aH0uaGlkZS10ZXh0e2ZvbnQ6MC8wIGE7Y29sb3I6dHJhbnNwYXJlbnQ7dGV4dC1zaGFkb3c6bm9uZTtiYWNrZ3JvdW5kLWNvbG9yOnRyYW5zcGFyZW50O2JvcmRlcjowfS5pbnB1dC1ibG9jay1sZXZlbHtkaXNwbGF5OmJsb2NrO3dpZHRoOjEwMCU7bWluLWhlaWdodDozMHB4Oy13ZWJraXQtYm94LXNpemluZzpib3JkZXItYm94Oy1tb3otYm94LXNpemluZzpib3JkZXItYm94O2JveC1zaXppbmc6Ym9yZGVyLWJveH0uYWN0aW9uY29sLC5ub3dyYXB7d2hpdGUtc3BhY2U6bm93cmFwfS5hY3Rpb25jb2wgYXt0ZXh0LWRlY29yYXRpb246bm9uZTtjb2xvcjojMDAwfS5hY3Rpb25jb2wgYS5kaXNhYmxlZHtjb2xvcjojY2NjO2N1cnNvcjpkZWZhdWx0fSNuYXZiYXIgLm5hdmJhci1pbm5lcntiYWNrZ3JvdW5kLWNvbG9yOiNlYmViZWI7YmFja2dyb3VuZC1pbWFnZTotbW96LWxpbmVhci1ncmFkaWVudCh0b3AsI2ZmZiwjY2NjKTtiYWNrZ3JvdW5kLWltYWdlOi13ZWJraXQtZ3JhZGllbnQobGluZWFyLDAgMCwwIDEwMCUsZnJvbSgjZmZmKSx0bygjY2NjKSk7YmFja2dyb3VuZC1pbWFnZTotd2Via2l0LWxpbmVhci1ncmFkaWVudCh0b3AsI2ZmZiwjY2NjKTtiYWNrZ3JvdW5kLWltYWdlOi1vLWxpbmVhci1ncmFkaWVudCh0b3AsI2ZmZiwjY2NjKTtiYWNrZ3JvdW5kLWltYWdlOmxpbmVhci1ncmFkaWVudCh0byBib3R0b20sI2ZmZiwjY2NjKTtiYWNrZ3JvdW5kLXJlcGVhdDpyZXBlYXQteDtmaWx0ZXI6cHJvZ2lkOkRYSW1hZ2VUcmFuc2Zvcm0uTWljcm9zb2Z0LmdyYWRpZW50KHN0YXJ0Q29sb3JzdHI9JyNmZmZmZmZmZicsIGVuZENvbG9yc3RyPScjZmZjY2NjY2MnLCBHcmFkaWVudFR5cGU9MCl9I25hdmJhciAubmF2YmFyLWlubmVyIC5icmFuZCwjbmF2YmFyIC5uYXZiYXItaW5uZXIgLm5hdj5saT5he3RleHQtc2hhZG93OjAgMXB4IDAgI2NjYztjb2xvcjojMzMzfSNuYXZiYXIgLm5hdmJhci1pbm5lciAuYnJhbmQgLmNhcmV0LCNuYXZiYXIgLm5hdmJhci1pbm5lciAubmF2PmxpPmEgLmNhcmV0e2JvcmRlci1ib3R0b20tY29sb3I6IzkzOTM5Mztib3JkZXItdG9wLWNvbG9yOiM5MzkzOTN9I25hdmJhciAubmF2YmFyLWlubmVyIC5icmFuZDpmb2N1cyAuY2FyODIgWTE1Mi44MzkgRS4wMDc3NgpHMSBYMTA0Ljc4NiBZMTUzLjEyNCBFLjAwNzQzCkcxIFgxMDQuNjYzIFkxNTMuMzgxIEUuMDA3NDIKRzEgWDEwNC4zMjggWTE1My44MDEgRS4wMTQKRzEgWDEwMy45MTYgWTE1NC4yNCBFLjAxNTY5CkcxIFgxMDMuNDE2IFkxNTQuNjcyIEUuMDE3MjIKRzEgWDEwMi42NSBZMTU1LjIwNyBFLjAyNDM0CkcxIFgxMDEuMTUxIFkxNTYuMTA5IEUuMDQ1NTgKRzEgWDEwMC43ODEgWTE1Ni4zODEgRS4wMTE5NwpHMSBYMTAwLjMyNyBZMTU2LjYyMyBFLjAxMzQKRzEgWDk5Ljg2NyBZMTU2LjY3NiBFLjAxMjA2CkcxIFg5OS41MDYgWTE1Ni44NSBFLjAxMDQ0CkcxIFg5OS4wODUgWTE1Ny4wMDcgRS4wMTE3MQpHMSBYOTguMDI0IFkxNTcuMjQzIEUuMDI4MzIKRzEgWDk3LjIwOCBZMTU3LjMxIEUuMDIxMzMKRzEgWDk2LjIwNyBZMTU3LjMyMiBFLjAyNjA4CkcxIFg5NC44NzYgWTE1Ny4yNCBFLjAzNDc1CkcxIFg5NC4zODggWTE1Ny4wOTQgRS4wMTMyNwpHMSBYOTMuODMgWTE1Ny4wMDEgRS4wMTQ3NApHMSBYOTMuNDE2IFkxNTYuODU0IEUuMDExNDUKRzEgWDkzLjA4MyBZMTU2Ljc2MiBFLjAwOQpHMSBYOTIuOTA1IFkxNTYuNjQyIEUuMDA1NTkKRzEgWDkyLjYzMiBZMTU2LjU0NyBFLjAwNzUzCkcxIFg5Mi40MzMgWTE1Ni4zNDQgRS4wMDc0MQpHMSBYOTIuMzQ0IFkxNTYuMDY2IEUuMDA3NjEKRzEgWDkyLjM3MiBZMTU1LjY0NyBFLjAxMDk0CkcxIFg5Mi4wOTkgWTE1NS41NzQgRS4wMDczNgpHMSBYOTEuOTA3IFkxNTUuNDIgRS4wMDY0MQpHMSBYOTEuODExIFkxNTUuMzA2IEUuMDAzODgKRzEgWDkxLjI2NSBZMTU1LjI4NCBFLjAxNDI0CkcxIFg5MC45OTggWTE1NS4xNTggRS4wMDc2OQpHMSBYOTAuNzc0IFkxNTQuOTYxIEUuMDA3NzcKRzEgWDkwLjM1MyBZMTU0LjgyNyBFLjAxMTUxCkcxIFg4OS45IFkxNTQuNTk5IEUuMDEzMjEKRzEgWDg5LjY3NSBZMTU0LjQzMiBFLjAwNzMKRzEgWDg5LjUzNiBZMTU0LjE0MyBFLjAwODM2CkcxIFg4OS41NTggWTE1My44MTcgRS4wMDg1MQpHMSBYODkuNjQzIFkxNTMuNTggRS4wMDY1NgpHMSBYODkuODY2IFkxNTMuMzI4IEUuMDA4NzcKRzEgWDkwLjEzNyBZMTUzLjIwOCBFLjAwNzcyCkcxIFg5MC40MDkgWTE1My4yMiBFLjAwNzA5CkcxIFg5MC44NjkgWTE1My40MjUgRS4wMTMxMgpHMSBYOTEuMDk0IFkxNTMuNTU5IEUuMDA2ODIKRzEgWDkxLjM4NyBZMTUzLjY0NiBFLjAwNzk2CkcxIFg5MS41MzUgWTE1My43NTkgRS4wMDQ4NQpHMSBYOTEuNzIzIFkxNTMuNzkgRS4wMDQ5NgpHMSBYOTEuOTE5IFkxNTMuODk2IEUuMDA1ODEKRzEgWDkyLjI2MyBZMTU0LjEyNCBFLjAxMDc1CkcxIFg5Mi41NTIgWTE1NC4xNDQgRS4wMDc1NQpHMSBYOTIuOTIzIFkxNTQuMjM4IEUuMDA5OTcKRzEgWDkzLjAwMSBZMTU0LjM0NSBFLjAwMzQ1CkcxIFg5My41MzIgWTE1NC4zOTMgRS4wMTM4OQpHMSBYOTMuODM3IFkxNTQuNTQ2IEUuMDA4ODkKRzEgWDk0LjI2NCBZMTU0LjU4NiBFLjAxMTE3CkcxIFg5NC41MDQgWTE1NC43MjkgRS4wMDcyOApHMSBYOTUuMjN2YXIgY29uZmlnPXRoaXMuY29uZmlnO2lmKHRoaXMudHJhY2tzJiZjb25maWcuYXV0b1N0YXJ0TG9hZCl7dGhpcy5zdGFydExvYWQoY29uZmlnLnN0YXJ0UG9zaXRpb24pO319O19wcm90by5vbk1lZGlhRGV0YWNoaW5nPWZ1bmN0aW9uIG9uTWVkaWFEZXRhY2hpbmcoKXt2YXIgbWVkaWE9dGhpcy5tZWRpYTtpZihtZWRpYSYmbWVkaWEuZW5kZWQpe2xvZ2dlclsibG9nZ2VyIl0ubG9nKCdNU0UgZGV0YWNoaW5nIGFuZCB2aWRlbyBlbmRlZCwgcmVzZXQgc3RhcnRQb3NpdGlvbicpO3RoaXMuc3RhcnRQb3NpdGlvbj10aGlzLmxhc3RDdXJyZW50VGltZT0wO30KaWYobWVkaWEpe21lZGlhLnJlbW92ZUV2ZW50TGlzdGVuZXIoJ3NlZWtpbmcnLHRoaXMub252c2Vla2luZyk7bWVkaWEucmVtb3ZlRXZlbnRMaXN0ZW5lcignZW5kZWQnLHRoaXMub252ZW5kZWQpO3RoaXMub252c2Vla2luZz10aGlzLm9udnNlZWtlZD10aGlzLm9udmVuZGVkPW51bGw7fQp0aGlzLm1lZGlhPXRoaXMubWVkaWFCdWZmZXI9dGhpcy52aWRlb0J1ZmZlcj1udWxsO3RoaXMubG9hZGVkbWV0YWRhdGE9ZmFsc2U7dGhpcy5mcmFnbWVudFRyYWNrZXIucmVtb3ZlQWxsRnJhZ21lbnRzKCk7dGhpcy5zdG9wTG9hZCgpO307X3Byb3RvLm9uQXVkaW9UcmFja3NVcGRhdGVkPWZ1bmN0aW9uIG9uQXVkaW9UcmFja3NVcGRhdGVkKGRhdGEpe2xvZ2dlclsibG9nZ2VyIl0ubG9nKCdhdWRpbyB0cmFja3MgdXBkYXRlZCcpO3RoaXMudHJhY2tzPWRhdGEuYXVkaW9UcmFja3M7fTtfcHJvdG8ub25BdWRpb1RyYWNrU3dpdGNoaW5nPWZ1bmN0aW9uIG9uQXVkaW9UcmFja1N3aXRjaGluZyhkYXRhKXt2YXIgYWx0QXVkaW89ISFkYXRhLnVybDt0aGlzLnRyYWNrSWQ9ZGF0YS5pZDt0aGlzLmZyYWdDdXJyZW50PW51bGw7dGhpcy5zdGF0ZT1TdGF0ZS5QQVVTRUQ7dGhpcy53YWl0aW5nRnJhZ21lbnQ9bnVsbDtpZighYWx0QXVkaW8pe2lmKHRoaXMuZGVtdXhlcil7dGhpcy5kZW11eGVyLmRlc3Ryb3koKTt0aGlzLmRlbXV4ZXI9bnVsbDt9fWVsc2V7dGhpcy5zZXRJbnRlcnZhbChhdWRpb19zdHJlYW1fY29udHJvbGxlcl9USUNLX0lOVEVSVkFMKTt9CmlmKGFsdEF1ZGlvKXt0aGlzLmF1ZGlvU3dpdGNoPXRydWU7dGhpcy5zdGF0ZT1TdGF0ZS5JRExFO30KdGhpcy50aWNrKCk7fTtfcHJvdG8ub25BdWRpb1RyYWNrTG9hZGVkPWZ1bmN0aW9uIG9uQXVkaW9UcmFja0xvYWRlZChkYXRhKXt2YXIgbmV3RGV0YWlscz1kYXRhLmRldGFpbHMsdHJhY2tJZD1kYXRhLmlkLHRyYWNrPXRoaXMudHJhY2tzW3RyYWNrSWRdLGR1cmF0aW9uPW5ld0RldGFpbHMudG90YWxkdXJhdGlvbixzbGlkaW5nPTA7bG9nZ2VyWyJsb2dnZXIiXS5sb2coInRyYWNrICIrdHJhY2tJZCsiIGxvYWRlZCBbIituZXdEZXRhaWxzLnN0YXJ0U04rIiwiK25ld0RldGFpbHMuZW5kU04rIl0sZHVyYXRpb246IitkdXJhdGlvbik7aWYobmV3RGV0YWlscy5saXZlKXt2YXIgY3VyRGV0YWlscz10cmFjay5kZXRhaWxzO2lmKGN1ckRldGFpbHMmJm5ld0RldGFpbHMuZnJhZ21lbnRzLmxlbmd0aD4wKVx1MDY0NiIsTToiXHUwNjI3XHUwNmNjXHUwNmE5IFx1MDY0NVx1MDYyN1x1MDZjMSIsTU06IiVkIFx1MDY0NVx1MDYyN1x1MDZjMSIseToiXHUwNjI3XHUwNmNjXHUwNmE5IFx1MDYzM1x1MDYyN1x1MDY0NCIseXk6IiVkIFx1MDYzM1x1MDYyN1x1MDY0NCJ9LHByZXBhcnNlOmZ1bmN0aW9uKGUpe3JldHVybiBlLnJlcGxhY2UoL1x1MDYwYy9nLCIsIil9LHBvc3Rmb3JtYXQ6ZnVuY3Rpb24oZSl7cmV0dXJuIGUucmVwbGFjZSgvLC9nLCJcdTA2MGMiKX0sd2Vlazp7ZG93OjEsZG95OjR9fSksTS5kZWZpbmVMb2NhbGUoInV6LWxhdG4iLHttb250aHM6IllhbnZhcl9GZXZyYWxfTWFydF9BcHJlbF9NYXlfSXl1bl9JeXVsX0F2Z3VzdF9TZW50YWJyX09rdGFicl9Ob3lhYnJfRGVrYWJyIi5zcGxpdCgiXyIpLG1vbnRoc1Nob3J0OiJZYW5fRmV2X01hcl9BcHJfTWF5X0l5dW5fSXl1bF9BdmdfU2VuX09rdF9Ob3lfRGVrIi5zcGxpdCgiXyIpLHdlZWtkYXlzOiJZYWtzaGFuYmFfRHVzaGFuYmFfU2VzaGFuYmFfQ2hvcnNoYW5iYV9QYXlzaGFuYmFfSnVtYV9TaGFuYmEiLnNwbGl0KCJfIiksd2Vla2RheXNTaG9ydDoiWWFrX0R1c2hfU2VzaF9DaG9yX1BheV9KdW1fU2hhbiIuc3BsaXQoIl8iKSx3ZWVrZGF5c01pbjoiWWFfRHVfU2VfQ2hvX1BhX0p1X1NoYSIuc3BsaXQoIl8iKSxsb25nRGF0ZUZvcm1hdDp7TFQ6IkhIOm1tIixMVFM6IkhIOm1tOnNzIixMOiJERC9NTS9ZWVlZIixMTDoiRCBNTU1NIFlZWVkiLExMTDoiRCBNTU1NIFlZWVkgSEg6bW0iLExMTEw6IkQgTU1NTSBZWVlZLCBkZGRkIEhIOm1tIn0sY2FsZW5kYXI6e3NhbWVEYXk6IltCdWd1biBzb2F0XSBMVCBbZGFdIixuZXh0RGF5OiJbRXJ0YWdhXSBMVCBbZGFdIixuZXh0V2VlazoiZGRkZCBba3VuaSBzb2F0XSBMVCBbZGFdIixsYXN0RGF5OiJbS2VjaGEgc29hdF0gTFQgW2RhXSIsbGFzdFdlZWs6IltPJ3RnYW5dIGRkZGQgW2t1bmkgc29hdF0gTFQgW2RhXSIsc2FtZUVsc2U6IkwifSxyZWxhdGl2ZVRpbWU6e2Z1dHVyZToiWWFxaW4gJXMgaWNoaWRhIixwYXN0OiJCaXIgbmVjaGEgJXMgb2xkaW4iLHM6InNvbml5YSIsc3M6IiVkIHNvbml5YSIsbToiYmlyIGRhcWlxYSIsbW06IiVkIGRhcWlxYSIsaDoiYmlyIHNvYXQiLGhoOiIlZCBzb2F0IixkOiJiaXIga3VuIixkZDoiJWQga3VuIixNOiJiaXIgb3kiLE1NOiIlZCBveSIseToiYmlyIHlpbCIseXk6IiVkIHlpbCJ9LHdlZWs6e2RvdzoxLGRveTo3fX0pLE0uZGVmaW5lTG9jYWxlKCJ1eiIse21vbnRoczoiXHUwNDRmXHUwNDNkXHUwNDMyXHUwNDMwXHUwNDQwX1x1MDQ0NFx1MDQzNVx1MDQzMlx1MDQ0MFx1MDQzMFx1MDQzYl9cdTA0M2NcdTA0MzBcdTA0NDBcdTA0NDJfXHUwNDMwXHUwNDNmXHUwNDQwXHUwNDM1XHUwNDNiX1x1MDQzY1x1MDQzMFx1MDQzOV9cdTA0MzhcdTA0NGVcdTA0M2RfXHUwNDM4XHUwNDRlXHUwNDNiX1x1MDQzMFx1MDQzMlx1MDQzM1x1MDQ0M1x1MDQ0MVx1MDQ0Ml9cdTA0NDFcdTA0MzVcdTA0M2RcdTA0NDJcdTA0NGZcdTA0MzFcdTA0NDBfXHUwNDNlXHUwNDNhXHUwMzE5Ljc5fSwgIndsYW4wIjogeyJyeF9ieXRlcyI6IDY5ODQ4NjE0LCAidHhfYnl0ZXMiOiAyNDY1ODIsICJyeF9wYWNrZXRzIjogMjYwNjg2LCAidHhfcGFja2V0cyI6IDIzODksICJyeF9lcnJzIjogMCwgInR4X2VycnMiOiAwLCAicnhfZHJvcCI6IDAsICJ0eF9kcm9wIjogMCwgImJhbmR3aWR0aCI6IDUwOC4zOX0sICJkb2NrZXIwIjogeyJyeF9ieXRlcyI6IDAsICJ0eF9ieXRlcyI6IDAsICJyeF9wYWNrZXRzIjogMCwgInR4X3BhY2tldHMiOiAwLCAicnhfZXJycyI6IDAsICJ0eF9lcnJzIjogMCwgInJ4X2Ryb3AiOiAwLCAidHhfZHJvcCI6IDAsICJiYW5kd2lkdGgiOiAwLjB9LCAiYzA5NWQ2YjRlM2UxIjogeyJyeF9ieXRlcyI6IDEzMTc1OCwgInR4X2J5dGVzIjogMTQ4ODY0OSwgInJ4X3BhY2tldHMiOiAxMTk3LCAidHhfcGFja2V0cyI6IDE0NjgsICJyeF9lcnJzIjogMCwgInR4X2VycnMiOiAwLCAicnhfZHJvcCI6IDAsICJ0eF9kcm9wIjogMCwgImJhbmR3aWR0aCI6IDAuMH0sICJ2ZXRoNjQyYjRlOSI6IHsicnhfYnl0ZXMiOiAxNDg1MTYsICJ0eF9ieXRlcyI6IDIzODE1NzEsICJyeF9wYWNrZXRzIjogMTE5NywgInR4X3BhY2tldHMiOiA0MTIzLCAicnhfZXJycyI6IDAsICJ0eF9lcnJzIjogMCwgInJ4X2Ryb3AiOiAwLCAidHhfZHJvcCI6IDAsICJiYW5kd2lkdGgiOiAwLjB9fSwgInN5c3RlbV9jcHVfdXNhZ2UiOiB7ImNwdSI6IDMxLjIyLCAiY3B1MCI6IDguNTEsICJjcHUxIjogNC4wNCwgImNwdTIiOiA5OS4wLCAiY3B1MyI6IDEwLjg5fSwgInN5c3RlbV9tZW1vcnkiOiB7InRvdGFsIjogNzgxOTk0OCwgImF2YWlsYWJsZSI6IDYzODc4MTIsICJ1c2VkIjogMTQzMjEzNn0sICJ3ZWJzb2NrZXRfY29ubmVjdGlvbnMiOiAzfV19PCFkb2N0eXBlIGh0bWw+CjxodG1sIGxhbmc9ImVuIj4KICAgIDxoZWFkPgogICAgICAgIDxtZXRhIGNoYXJzZXQ9InV0Zi04IiAvPgogICAgICAgIDxtZXRhIGh0dHAtZXF1aXY9IlgtVUEtQ29tcGF0aWJsZSIgY29udGVudD0iSUU9ZWRnZSIgLz4KICAgICAgICA8bWV0YSBuYW1lPSJ2aWV3cG9ydCIgY29udGVudD0id2lkdGg9ZGV2aWNlLXdpZHRoLGluaXRpYWwtc2NhbGU9MS4wIiAvPgogICAgICAgIDx0aXRsZT5NYWluc2FpbDwvdGl0bGU+CiAgICAgICAgPG1ldGEgbmFtZT0iZGVzY3JpcHRpb24iIGNvbnRlbnQ9Ik1haW5zYWlsIGlzIHRoZSBwb3B1bGFyIHdlYiBpbnRlcmZhY2UgZm9yIEtsaXBwZXIuIiAvPgoKICAgICAgICA8bGluayByZWw9Imljb24iIHR5cGU9ImltYWdlL3BuZyIgc2l6ZXM9IjMyeDMyIiBocmVmPSIvaW1nL2ljb25zL2Zhdmljb24tMzJ4MzIucG5nIiAvPgogICAgICAgIDxsaW5rIHJlbD0iaWNvbiIgdHlwZT0iaW1hZ2UvcG5nIiBzaXplcz0iMTZ4MTYiIGhyZWY9Ii9pbWcvaWNvbnMvZmF2aWNvbi0xNngxNi5wbmciIC8+CiAgICAgICAgPG1ldGEgbmFtZT0idGhlbWUtY29sb3IiIGNvbnRlbnQ9IiMxMjEyMTIiIC8+CiAgICAgICAgPG1ldGEgbmFtZT0ibW9iaWxlLXdlYi1hcHAtY2FwYWJsZSIgY29udGVudD0ieWVzIiAvPgoxMTEKRzEgWDE2MC43MTUgWTQ1LjY3MiBFLjM2MTc0CkcxIFgxNjEuMjA3IFk0NS43NzEgRS4wMTMwOApHMSBYMTUxLjE4MyBZNTUuNzk1IEUuMzY5MzcKRzEgWDE1MS4zOTUgWTU2LjE3MyBFLjAxMTI5CkcxIFgxNjEuMzAyIFk0Ni4yNjYgRS4zNjUwNgpHMSBYMTYxLjQ3NCBZNDYuNjg2IEUuMDExODMKRzEgWDE1MS40NzkgWTU2LjY4MSBFLjM2ODMKRzEgWDE1MS4zNzkgWTU2Ljc1NSBFLjAwMzI0CjtXSURUSDowLjQxMTA5NQpHMSBYMTUxLjI3OSBZNTYuODI4IEUuMDAyOTMKO1dJRFRIOjAuMzcyMTkKRzEgWDE1MS4xMjYgWTU3LjI2NiBFLjAwOTg0CjtXSURUSDowLjQ0OTk5OQpHMSBYMTUxLjM0MSBZNTcuNDA5IEUuMDA2NzMKRzEgWDE2MS42ODQgWTQ3LjA2NyBFLjM4MTEKRzEgWDE2MS45MTMgWTQ3LjQyOSBFLjAxMTE2CkcxIFgxNTEuNTM2IFk1Ny44MDYgRS4zODIzOApHMSBYMTUxLjczNiBZNTguMTk3IEUuMDExNDQKRzEgWDE2Mi4xNTIgWTQ3Ljc4IEUuMzgzODMKRzEgWDE2Mi40MzMgWTQ4LjA5IEUuMDEwOQpHMSBYMTUxLjk0MiBZNTguNTgxIEUuMzg2NTgKRzEgWDE1Mi4xNDggWTU4Ljk2NiBFLjAxMTM4CkcxIFgxNjIuNzE5IFk0OC4zOTUgRS4zODk1MgpHMSBYMTYzLjAyOCBZNDguNjc3IEUuMDEwOQpHMSBYMTUyLjM0OSBZNTkuMzU1IEUuMzkzNDkKRzEgWDE1Mi40OTEgWTU5LjgwNCBFLjAxMjI3CkcxIFgxNjMuNDE1IFk0OC44ODEgRS40MDI1MQpHMSBYMTYzLjg4NSBZNDkuMDAyIEUuMDEyNjUKRzEgWDE1Mi42NTIgWTYwLjIzNCBFLjQxMzkKRzEgWDE1Mi44MSBZNjAuNjY3IEUuMDEyMDEKRzEgWDE1OS4wNTMgWTU0LjQyNCBFLjIzMDA0CkcxIFgxNTkuMTUyIFk1NC4yOTEgRS4wMDQzMgo7V0lEVEg6MC40MTEwOTUKRzEgWDE1OS4yNSBZNTQuMTU5IEUuMDAzODgKO1dJRFRIOjAuMzcyMTkKRzEgWDE1OS4yMDkgWTU0LjE3IEUuMDAwOQpHMSBYMTU5LjE5IFk1NC4wOTkgRS4wMDE1NgpHMSBYMTU5LjI2MSBZNTQuMTE4IEUuMDAxNTYKRzEgWDE1OS4wNjggWTU1IEYxMDgwMAo7V0lEVEg6MC40NDk5OTkKRzEgRjQ4MDAKRzEgWDE1Mi45NTEgWTYxLjExNyBFLjIyNTQKRzEgWDE1My4xMDQgWTYxLjU1NSBFLjAxMjA5CkcxIFgxNTkuMjAxIFk1NS40NTggRS4yMjQ2NgpHMSBYMTU5LjQyMiBZNTUuODI4IEUuMDExMjMKRzEgWDE1My4yMjUgWTYyLjAyNSBFLjIyODM1CkcxIFgxNTMuMjgxIFk2Mi41NiBFLjAxNDAyCkcxIFgxNTkuNjM3IFk1Ni4yMDQgRS4yMzQyMQpHMSBYMTU5Ljg0MiBZNTYuNTkgRS4wMTEzOQpHMSBYMTUzLjM1NCBZNjMuMDc4IEUuMjM5MDcKRzEgWDE1My4zODQgWTYzLjYzOSBFLjAxNDY0CkcxIFgxNjAuMDI5IFk1Ni45OTMgRS4yNDQ4OApHMSBYMTYwLjIyOSBZNTcuMzg1IEUuMDExNDcKRzEgWDE1My4zODcgWTY0LjIyNyBFLjI1MjEyCkcxIFgxNTMuNDMxIFk2NC43NzQgRS4wMTQzCkcxIFgxNjAuMzkgWTU3LjgxNSBFLjI1NjQzCkcxIFgxNjAuNTY1IFk1OC4yMyBFLjAxMTc0CkcxIFgxNTMuMzI3IFk2NS40NjggRS4yNjY3MQpHMSBYMTUzLjI3MyBZNjUuNjA0IDMKRzEgWDc1LjUzMyBZMTY2LjU5NyBFLjAxMDc3CkcxIFg3NC45MTIgWTE2Ni40NDMgRS4wMTY2NwpHMSBYNzQuNjU1IFkxNjYuMjUyIEUuMDA4MzQKRzEgWDc0LjQzOCBZMTY2LjE5OSBFLjAwNTgyCkcxIFg3My45NTIgWTE2NS45ODUgRS4wMTM4NApHMSBYNzMuNzg5IFkxNjUuODkzIEUuMDA0ODgKRzEgWDczLjE1MSBZMTY1LjcyNiBFLjAxNzE4CkcxIFg3Mi42MDMgWTE2NS42MTggRS4wMTQ1NQpHMSBYNzIuMjc2IFkxNjUuNDM3IEUuMDA5NzQKRzEgWDcxLjM4NyBZMTY1LjE1MyBFLjAyNDMyCkcxIFg3MS4xMzcgWTE2NS4wMTEgRS4wMDc0OQpHMSBYNzAuNDk0IFkxNjQuODc1IEUuMDE3MTIKRzEgWDcwLjA4OSBZMTY0LjY3OSBFLjAxMTcyCkcxIFg2OS45NyBZMTY0LjQyOSBFLjAwNzIxCkcxIFg2OS44OTkgWTE2My44MjMgRS4wMTU5CkcxIFg3MC4wMDYgWTE2Mi45NzYgRS4wMjIyNApHMSBYNzAuMTg1IFkxNjEuOTU4IEUuMDI2OTMKRzEgWDcwLjM4OSBZMTYxLjMyOCBFLjAxNzI1CkcxIFg3MC42ODIgWTE2MS4wMTYgRS4wMTExNQpHMSBYNzEuMDM5IFkxNjAuOTE2IEUuMDA5NjYKRzEgWDcxLjExMyBZMTYwLjk0MiBFLjAwMjA0CkcxIFg3MS4xOTggWTE2MS4wMzUgRS4wMDMyOApHMSBYNzEuMzc0IFkxNjEuMzc2IEUuMDEKRzEgWDcxLjYyMiBZMTYxLjczNiBFLjAxMTM5CkcxIFg3MS44NDEgWTE2MS4yMTkgRS4wMTQ2MwpHMSBYNzIuMDA4IFkxNjAuNTY1IEUuMDE3NTkKRzEgWDcyLjI0NSBZMTYwLjEyOCBFLjAxMjk1CkcxIFg3Mi40MyBZMTU5LjcwOCBFLjAxMTk2CkcxIFg3Mi40NDYgWTE1OS4yNTQgRS4wMTE4NApHMSBYNzIuMzM4IFkxNTkuMDI4IEUuMDA2NTMKRzEgWDcyLjI3MyBZMTU4LjUwNCBFLjAxMzc2CkcxIFg3Mi4yMTQgWTE1OC4zNzggRS4wMDM2MwpHMSBYNzIuMTU4IFkxNTguMTM0IEUuMDA2NTIKRzEgWDcxLjk5MiBZMTU3LjY4NiBFLjAxMjQ1CkcxIFg3MS44OCBZMTU3LjIwOCBFLjAxMjc5CkcxIFg3MS43ODMgWTE1Ny4wMzYgRS4wMDUxNQpHMSBYNzEuNzI3IFkxNTYuNjY4IEUuMDA5NwpHMSBYNzEuNTgxIFkxNTYuMzY1IEUuMDA4NzYKRzEgWDcxLjQxMiBZMTU1Ljg2NCBFLjAxMzc4CkcxIFg3MS4xNjcgWTE1NS4zNCBFLjAxNTA3CkcxIFg3MC45NTkgWTE1NC44MjMgRS4wMTQ1MgpHMSBYNzAuOTQ4IFkxNTQuNjgxIEUuMDAzNzEKRzEgWDcwLjYyMiBZMTU0LjI1OCBFLjAxMzkxCkcxIFg3MC40ODQgWTE1My44OTcgRS4wMTAwNwpHMSBYNzAuNTU5IFkxNTMuNjY2IEUuMDA2MzMKRzEgWDcwLjY0MyBZMTUzLjUyNyBFLjAwNDIzCkcxIFg2OS45NTcgWTE1My4xNDYgRS4wMjA0NQpHMSBYNjkuNzY0IFkxNTIuODIgRS4wMDk4NwpHMSBYNjkuNzk5IFkxNTIuNjg0IEUuMDAzNjYKRzEgWDY5Ljg4NyBZMTUyLjU0OCBFLjAwNDIyCkcxIFg2OS45MjEgWTE1Mi4zNzUgRS4wMDQ1OQpHMSBYNjkuNDkyIFkxNTIuMjQ4IEUuMDExNjYKRzEgWDY5LjA5IFkxNTIuMTYzIEUuMDEwNzEKRzEgWDY4LjgxNiBZMTUxLjg1MSBFLjAxMDgyCkcxIFg2OC44MjMgWTE1MS43MTggRS4wMDM0NwpHMTQ2LjU0MiBFLjAxMTU0CkcxIFgyMjEuMTc5IFkyMDMuOTc5IEUyLjExNjQ2CkcxIFgyMjEuNzcgWTIwMy45NzkgRS4wMTU0CkcxIFgxNjQuMTUyIFkxNDYuMzYxIEUyLjEyMzEzCkcxIFgxNjQuNTQ0IFkxNDYuMTYyIEUuMDExNDUKRzEgWDIyMi4zNjEgWTIwMy45NzkgRTIuMTMwNDYKRzEgWDIyMi45NTIgWTIwMy45NzkgRS4wMTU0CkcxIFgxNjQuOTI3IFkxNDUuOTU0IEUyLjEzODEzCkcxIFgxNjUuMzA5IFkxNDUuNzQ1IEUuMDExMzUKRzEgWDIyMy41NDMgWTIwMy45NzkgRTIuMTQ1ODMKRzEgWDIyNC4xMjIgWTIwNC4wNiBFLjAxNTIzCjtXSURUSDowLjM3MjE5CkcxIFgyMjQuMTMzIFkyMDQuMTAxIEUuMDAwOQpHMSBYMjI0LjA2MSBZMjA0LjEyIEUuMDAxNTgKRzEgWDIyNC4wOCBZMjA0LjA0OSBFLjAwMTU2CjtXSURUSDowLjQxMTA5NQpHMSBYMjI0LjAzIFkyMDMuOTM4IEUuMDAyODgKO1dJRFRIOjAuNDQ5OTk5CkcxIFgyMjMuOTggWTIwMy44MjYgRS4wMDMyCkcxIFgxNjUuNjg0IFkxNDUuNTI5IEUyLjE0ODEzCkcxIFgxNjYuMDM5IFkxNDUuMjk0IEUuMDExMDkKRzEgWDIyMy45OCBZMjAzLjIzNSBFMi4xMzUwMwpHMSBYMjIzLjk4IFkyMDIuNjQ0IEUuMDE1NApHMSBYMTY2LjM4OCBZMTQ1LjA1MiBFMi4xMjIxNwpHMSBYMTY2Ljc0NCBZMTQ0LjgxNyBFLjAxMTExCkcxIFgyMjMuOTggWTIwMi4wNTMgRTIuMTA5MDUKRzEgWDIyMy45OCBZMjAxLjQ2MyBFLjAxNTM3CkcxIFgxNjcuMTQxIFkxNDQuNjIzIEUyLjA5NDQ0CkcxIFgxNjcuNDk4IFkxNDQuMzkgRS4wMTExMQpHMSBYMjIzLjk4IFkyMDAuODcyIEUyLjA4MTI3CkcxIFgyMjMuOTggWTIwMC4yODEgRS4wMTU0CkcxIFgxNjcuODU4IFkxNDQuMTU5IEUyLjA2OApHMSBYMTY4LjIxNCBZMTQzLjkyMyBFLjAxMTEzCkcxIFgyMjMuOTggWTE5OS42OSBFMi4wNTQ5CkcxIFgyMjMuOTggWTE5OS4wOTkgRS4wMTU0CkcxIFgxNjguNTMyIFkxNDMuNjUxIEUyLjA0MzE3CkcxIFgxNjguODUgWTE0My4zNzcgRS4wMTA5NApHMSBYMjIzLjk4IFkxOTguNTA4IEUyLjAzMTQ3CkcxIFgyMjMuOTggWTE5Ny45MTcgRS4wMTU0CkcxIFgxNjkuMTU2IFkxNDMuMDkzIEUyLjAyMDE3CkcxIFgxNjkuNDY1IFkxNDIuODExIEUuMDEwOQpHMSBYMjIzLjk4IFkxOTcuMzI2IEUyLjAwODc5CkcxIFgyMjMuOTggWTE5Ni43MzYgRS4wMTUzNwpHMSBYMTY5Ljc1MSBZMTQyLjUwNiBFMS45OTgyNwpHMSBYMTY5Ljk2NiBZMTQyLjEzIEUuMDExMjkKRzEgWDIyMy45OCBZMTk2LjE0NSBFMS45OTAzNQpHMSBYMjIzLjk4IFkxOTUuNTU0IEUuMDE1NApHMSBYMTcwLjEwOCBZMTQxLjY4MSBFMS45ODUxMQpHMSBYMTcwLjExOCBZMTQxLjEwMSBFLjAxNTExCkcxIFgyMjMuOTggWTE5NC45NjMgRTEuOTg0NzMKRzEgWDIyMy45ODEgWTE5NC4zNzIgRS4wMTU0CkcxIFgxNjkuNjg3IFkxNDAuMDc5IEUyLjAwMDYzCjtXSVBFX1NUQVJUCkcxIEY4NjQwCkcxIFgxNzIuMTM3IFkxNDIuNTI5IEUtLjgKO1dJUEVfRU5ECkcxIFoxLjIgRjcyMApHMSBYMTY4LjcwMyBZMTM4LjUwNCBGMTBkPSJzZXR0aW5nc19wbHVnaW5fZXJyb3J0cmFja2luZyIKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGNsYXNzPSJ0YWItcGFuZSAgIgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgPgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIDxoMz5FcnJvciBUcmFja2luZzwvaDM+CgoKPHA+CiAgICBFcnJvciB0cmFja2luZyB3aWxsIGNhdXNlIGFueSBsb2dnZWQgZXhjZXB0aW9ucyBpbiB0aGUgc2VydmVyIGFuZCB0aGUgYnJvd3NlciBpbnRlcmZhY2UgdG8gYmUgc2VudAogICAgdG8gT2N0b1ByaW50J3MgPGEgaHJlZj0iaHR0cHM6Ly9zZW50cnkuaW8vIiB0YXJnZXQ9Il9ibGFuayIgcmVsPSJub29wZW5lciBub3JlZmVyZXIiPlNlbnRyeSBpbnN0YW5jZTwvYT4uCjwvcD4KPHA+CiAgICBCeSBlbmFibGluZyBpdCB5b3UgaGVscCB0byBnYXRoZXIgZGV0YWlsZWQgaW5mb3JtYXRpb24gb24gdGhlIGNhdXNlIG9mIGJ1Z3Mgb3Igb3RoZXIgaXNzdWVzLiBUaGlzIGlzIGVzcGVjaWFsbHkgdmFsdWFibGUKICAgIG9uIHJlbGVhc2UgY2FuZGlkYXRlcywgd2hpY2ggaXMgd2h5IHRoaXMgcGx1Z2luIHdpbGwgYWxzbyBwcm9tcHQgeW91IHRvIGVuYWJsZSBlcnJvciB0cmFja2luZyBpZiBpdCBkZXRlY3RzIHRoYXQKICAgIHlvdSBhcmUgc3Vic2NyaWJlZCB0byBhbiBSQyByZWxlYXNlIGNoYW5uZWwuCjwvcD4KPHA+CiAgICBQbGVhc2UgZW5hYmxlIHRoaXMgaWYgeW91IGFyZSBydW5uaW5nIGEgcmVsZWFzZSBjYW5kaWRhdGUsIG9yIGFyZSBwcm9tcHRlZCB0byBkbyBzbyB3aGlsZSBkZWJ1Z2dpbmcgYW4gaXNzdWUgdGhhdCB5b3UgcmVwb3J0ZWQuCjwvcD4KCgo8Zm9ybSBjbGFzcz0iZm9ybS1ob3Jpem9udGFsIiBvbnN1Ym1pdD0icmV0dXJuIGZhbHNlOyI+CiAgICA8ZGl2IGNsYXNzPSJjb250cm9sLWdyb3VwIj4KICAgICAgICA8ZGl2IGNsYXNzPSJjb250cm9scyI+CiAgICAgICAgICAgIDxsYWJlbCBjbGFzcz0iY2hlY2tib3giPgogICAgICAgICAgICAgICAgPGlucHV0IHR5cGU9ImNoZWNrYm94IiBkYXRhLWJpbmQ9ImNoZWNrZWQ6IHNldHRpbmdzLnBsdWdpbnMuZXJyb3J0cmFja2luZy5lbmFibGVkIj4gRW5hYmxlIGVycm9yIHRyYWNraW5nCiAgICAgICAgICAgIDwvbGFiZWw+CiAgICAgICAgICAgIDxzcGFuIGNsYXNzPSJoZWxwLWJsb2NrIj5QbGVhc2UgcmVsb2FkIHRoZSBwYWdlIGFmdGVyIGVuYWJsaW5nL2Rpc2FibGluZyAmIHNhdmluZzwvc3Bhbj4KICAgICAgICA8L2Rpdj4KICAgIDwvZGl2PgogICAgPGRpdiBjbGFzcz0iY29udHJvbC1ncm91cCI+CiAgICAgICAgPGxhYmVsIGNsYXNzPSJjb250cm9sLWxhYmVsIiB0aXRsZT0iUmFuZG9tIHVuaXF1ZSBpbnN0YW5jZSBJRCI+SW5zdGFuY2UgSUQ8L2xhYmVsPgogICAgICAgIDxkaXYgY2xhc3M9ImNvbnRyb2xzIj4KICAgICAgICAgICAgPHNwYW4gc3R5bGU9InBhZGRpbmctdG9wOmxpbmsucm91dGVyLWxpbmstYWN0aXZle2JhY2tncm91bmQ6cmdiYSgyNTUsMjU1LDI1NSwuMil9bmF2IHVsLm5hdmk+bGk+dWwuY2hpbGQgLm5hdi1saW5rPnNwYW4ubmF2LXRpdGxle3RleHQtdHJhbnNmb3JtOmNhcGl0YWxpemU7Zm9udC13ZWlnaHQ6NDAwO2ZvbnQtc2l6ZToxNHB4fS51c2VyLXNlbGVjdC1ub25ley13ZWJraXQtdG91Y2gtY2FsbG91dDpub25lOy13ZWJraXQtdXNlci1zZWxlY3Q6bm9uZTsta2h0bWwtdXNlci1zZWxlY3Q6bm9uZTstbW96LXVzZXItc2VsZWN0Om5vbmU7LW1zLXVzZXItc2VsZWN0Om5vbmU7dXNlci1zZWxlY3Q6bm9uZX0ubWFpbnNhaWwtZWRpdG9ye2hlaWdodDo5MiU7d2lkdGg6MTAwJTtvdmVyZmxvdzpoaWRkZW47LXdlYmtpdC1vdmVyZmxvdy1zY3JvbGxpbmc6dG91Y2g7dXNlci1zZWxlY3Q6YXV0bzstd2Via2l0LXVzZXItc2VsZWN0OmF1dG99LmVjaGFydHMtdG9vbHRpcHt6LWluZGV4OjUhaW1wb3J0YW50fS50b29sdGlwX19jb250ZW50LW9wYWNpdHkxe29wYWNpdHk6MSFpbXBvcnRhbnR9LnYtYnRuLm1pbndpZHRoLTB7bWluLXdpZHRoOmF1dG8haW1wb3J0YW50fS5taW5IZWlnaHQzNnttaW4taGVpZ2h0OjM2cHh9LmN1cnNvci1wb2ludGVye2N1cnNvcjpwb2ludGVyIWltcG9ydGFudH0uaWNvbi1yb3RhdGV7YW5pbWF0aW9uLW5hbWU6c3BpbjthbmltYXRpb24tZHVyYXRpb246MXM7YW5pbWF0aW9uLWl0ZXJhdGlvbi1jb3VudDppbmZpbml0ZTthbmltYXRpb24tdGltaW5nLWZ1bmN0aW9uOmxpbmVhcn0udi10b2FzdHtwYWRkaW5nOjhweCFpbXBvcnRhbnR9QGtleWZyYW1lcyBzcGluezAle3RyYW5zZm9ybTpyb3RhdGUoMCl9dG97dHJhbnNmb3JtOnJvdGF0ZSgtMzYwZGVnKX19QG1lZGlhIHNjcmVlbiBhbmQgKG1heC13aWR0aDogNDgwcHgpey52LWFwcGxpY2F0aW9uLS1pcy1sdHIgLnVwZGF0ZU1hbmFnZXIudi10aW1lbGluZS0tZGVuc2U6bm90KC52LXRpbWVsaW5lLS1yZXZlcnNlKTpiZWZvcmV7bGVmdDoyM3B4fS51cGRhdGVNYW5hZ2VyIC52LXRpbWVsaW5lLWl0ZW1fX2RpdmlkZXJ7bWluLXdpZHRoOjQ4cHh9LnVwZGF0ZU1hbmFnZXIudi10aW1lbGluZS0tZGVuc2UgLnYtdGltZWxpbmUtaXRlbV9fYm9keXttYXgtd2lkdGg6Y2FsYygxMDAlIC0gNDhweCl9fTpyb290ey0tYXBwLWhlaWdodDogMTAwJX0jY29udGVudHtiYWNrZ3JvdW5kLWF0dGFjaG1lbnQ6Zml4ZWQ7YmFja2dyb3VuZC1zaXplOmNvdmVyO2JhY2tncm91bmQtcmVwZWF0Om5vLXJlcGVhdH0udi1idG46bm90KC52LWJ0bi0tb3V0bGluZWQpLnByaW1hcnl7Y29sb3I6dmFyKC0tdi1idG4tdGV4dC1wcmltYXJ5KX1Aa2V5ZnJhbWVzIGZhZGVPdXR7MCV7b3BhY2l0eToxfXRve29wYWNpdHk6MH19LnYtdG9hc3QtLWZhZGUtb3V0e2FuaW1hdGlvbi1uYW1lOmZhZGVPdXR9QGtleWZyYW1lcyBmYWRlSW5Eb3duezAle29wYWNpdHk6MDt0cmFuc2Zvcm06dHJhbnNsYXRlM2QoMCwtMTAwJSwwKX10b3tvcGFjaXR5OjE7dHJhbnNmb3JtOm5vbmV9fS52LXRvYXN0LS1mYWRlLWluLWRvd257dHVybiBNYXRoLnJvdW5kKHIubGVuZ3RoLzIpfTtmdW5jdGlvbiBYXyhyKXtyZXR1cm57c2VyaWVzVHlwZTpyLHJlc2V0OmZ1bmN0aW9uKHQsZSxpKXt2YXIgbj10LmdldERhdGEoKSxhPXQuZ2V0KCJzYW1wbGluZyIpLG89dC5jb29yZGluYXRlU3lzdGVtLHM9bi5jb3VudCgpO2lmKHM+MTAmJm8udHlwZT09PSJjYXJ0ZXNpYW4yZCImJmEpe3ZhciBsPW8uZ2V0QmFzZUF4aXMoKSx1PW8uZ2V0T3RoZXJBeGlzKGwpLGY9bC5nZXRFeHRlbnQoKSxoPWkuZ2V0RGV2aWNlUGl4ZWxSYXRpbygpLGM9TWF0aC5hYnMoZlsxXS1mWzBdKSooaHx8MSksdj1NYXRoLnJvdW5kKHMvYyk7aWYoaXNGaW5pdGUodikmJnY+MSl7YT09PSJsdHRiIiYmdC5zZXREYXRhKG4ubHR0YkRvd25TYW1wbGUobi5tYXBEaW1lbnNpb24odS5kaW0pLDEvdikpO3ZhciBkPXZvaWQgMDt6KGEpP2Q9Y0lbYV06WChhKSYmKGQ9YSksZCYmdC5zZXREYXRhKG4uZG93blNhbXBsZShuLm1hcERpbWVuc2lvbih1LmRpbSksMS92LGQsZEkpKX19fX19ZnVuY3Rpb24gT1Iocil7ci5yZWdpc3RlckNoYXJ0VmlldyhoSSksci5yZWdpc3RlclNlcmllc01vZGVsKFdMKSxyLnJlZ2lzdGVyTGF5b3V0KHZJKCJsaW5lIiwhMCkpLHIucmVnaXN0ZXJWaXN1YWwoe3Nlcmllc1R5cGU6ImxpbmUiLHJlc2V0OmZ1bmN0aW9uKHQpe3ZhciBlPXQuZ2V0RGF0YSgpLGk9dC5nZXRNb2RlbCgibGluZVN0eWxlIikuZ2V0TGluZVN0eWxlKCk7aSYmIWkuc3Ryb2tlJiYoaS5zdHJva2U9ZS5nZXRWaXN1YWwoInN0eWxlIikuZmlsbCksZS5zZXRWaXN1YWwoImxlZ2VuZExpbmVTdHlsZSIsaSl9fSksci5yZWdpc3RlclByb2Nlc3NvcihyLlBSSU9SSVRZLlBST0NFU1NPUi5TVEFUSVNUSUMsWF8oImxpbmUiKSl9dmFyIFpfPWZ1bmN0aW9uKHIpe0IodCxyKTtmdW5jdGlvbiB0KCl7dmFyIGU9ciE9PW51bGwmJnIuYXBwbHkodGhpcyxhcmd1bWVudHMpfHx0aGlzO3JldHVybiBlLnR5cGU9dC50eXBlLGV9cmV0dXJuIHQucHJvdG90eXBlLmdldEluaXRpYWxEYXRhPWZ1bmN0aW9uKGUsaSl7cmV0dXJuIGV2KG51bGwsdGhpcyx7dXNlRW5jb2RlRGVmYXVsdGVyOiEwfSl9LHQucHJvdG90eXBlLmdldE1hcmtlclBvc2l0aW9uPWZ1bmN0aW9uKGUsaSxuKXt2YXIgYT10aGlzLmNvb3JkaW5hdGVTeXN0ZW07aWYoYSYmYS5jbGFtcERhdGEpe3ZhciBvPWEuY2xhbXBEYXRhKGUpLHM9YS5kYXRhVG9Qb2ludChvKTtpZihuKUEoYS5nZXRBeGVzKCksZnVuY3Rpb24oYyx2KXtpZihjLnR5cGU9PT0iY2F0ZWdvcnkiJiZpIT1udWxsKXt2YXIgZD1jLmdldFRpY2tzQ29vcmRzKCksZz1vW3ZdLHA9aVt2XT09PSJ4MSJ8fGlbdl09PT0ieTEiO2lmKHAmJihnKz0xKSxkLmxlbmd0aDwyKXJldHVybjtpZihkLmxlbmd0aD09PTIpe3Nbdl09Yy50b0dsb2JhbENvb3JkKGMuZ2V0RXh0ZW50KClbcD8xOjBdKTtyZXR1cm59Zm9yKHZhciB5PXZvaWQgMCxtPXZvaWQgMCxfPTEsUz0wO1M8ZC5sZW5ndGg7UysrKXt2YXIgYj1kW1NdLmNvb3JkLHc9Uz09PWQubGVuZ3RoLTE/ZFtTLTFdLnRpY2tWYWx1ZSttaWx5OlJvYm90byxzYW5zLXNlcmlmO2xpbmUtaGVpZ2h0OjEuNTtkaXNwbGF5OmZsZXg7ZmxleDoxIDEgYXV0bztwb3NpdGlvbjphYnNvbHV0ZTt0b3A6LTEwMDBweDtyaWdodDowO3dpZHRoOmF1dG87bWluLXdpZHRoOjE3MHB4O2hlaWdodDo0MHB4O2FsaWduLWl0ZW1zOmNlbnRlcjtwYWRkaW5nOjAgMTZweCAwIDMycHg7Ym9yZGVyLXJhZGl1czo2cHg7dGV4dC13cmFwOm5vd3JhcH0udi1pbnB1dC0tdGV4dC1yaWdodCBpbnB1dHt0ZXh0LWFsaWduOnJpZ2h0fS52LWlucHV0LS13aWR0aC14LXNtYWxse3dpZHRoOjEwMHB4fS52LWlucHV0LS13aWR0aC1zbWFsbHt3aWR0aDoxNDBweH0udi1pbnB1dC0td2lkdGgtbWVkaXVte3dpZHRoOjE4MHB4fS52LWlucHV0LS14LWRlbnNlIC52LWlucHV0X19zbG90e21pbi1oZWlnaHQ6MjVweCFpbXBvcnRhbnR9LnYtaW5wdXQtLWRlbnNlIC52LWlucHV0X19zbG90e21pbi1oZWlnaHQ6MzZweCFpbXBvcnRhbnR9LnYtaW5wdXQtLXgtZGVuc2UgLnYtc2VsZWN0X19zZWxlY3Rpb25ze3BhZGRpbmc6MnB4IDAhaW1wb3J0YW50fS52LWlucHV0LS14LWRlbnNlIC52LWlucHV0X19hcHBlbmQtaW5uZXJ7bWFyZ2luLXRvcDo2cHghaW1wb3J0YW50fS52LXRleHQtZmllbGQudi10ZXh0LWZpZWxkLS1zb2xvIC52LWlucHV0X19jb250cm9se21pbi1oZWlnaHQ6MTBweH0udi1pbnB1dC0tZGVuc2UgLnYtdGV4dC1maWVsZC0tYm94IC52LWlucHV0X19zbG90LC52LXRleHQtZmllbGQtLW91dGxpbmUgLnYtaW5wdXRfX3Nsb3R7bWluLWhlaWdodDozMHB4IWltcG9ydGFudH0udi1saXN0LWl0ZW0tLXgtZGVuc2UgLnYtbGlzdC1pdGVtX19jb250ZW50LC52LWxpc3QtLXgtZGVuc2UgLnYtbGlzdC1pdGVtIC52LWxpc3QtaXRlbV9fY29udGVudHtwYWRkaW5nOjZweCAwIWltcG9ydGFudH0udi1saXN0LWl0ZW0tLXgtZGVuc2UgLnYtbGlzdC1pdGVtX19hY3Rpb24sLnYtbGlzdC0teC1kZW5zZSAudi1saXN0LWl0ZW0gLnYtbGlzdC1pdGVtX19hY3Rpb257bWFyZ2luOjhweCAwfS52LXRleHQtZmllbGQtb3V0ZXItYnRuIC52LWlucHV0X19jb250cm9se21pbi1oZWlnaHQ6MzZweCFpbXBvcnRhbnR9LnYtdGV4dC1maWVsZC1vdXRlci1idG4gLnYtaW5wdXRfX3ByZXBlbmQtb3V0ZXIsLnYtdGV4dC1maWVsZC1vdXRlci1idG4gLnYtaW5wdXRfX2FwcGVuZC1vdXRlcnttYXJnaW4tdG9wOjAhaW1wb3J0YW50O21hcmdpbi1ib3R0b206MCFpbXBvcnRhbnR9LnRoZW1lLS1saWdodC52LWxpc3QtaXRlbS0tZGlzYWJsZWQgLnYtbGlzdC1pdGVtX19pY29uIC52LWljb257Y29sb3I6IzAwMDAwMDYxIWltcG9ydGFudH0udGhlbWUtLWRhcmsudi1saXN0LWl0ZW0tLWRpc2FibGVkIC52LWxpc3QtaXRlbV9faWNvbiAudi1pY29ue2NvbG9yOiNmZmZmZmY4MCFpbXBvcnRhbnR9LnRoZW1lLS1saWdodC52LWRhdGEtdGFibGUubm8taG92ZXI+LnYtZGF0YS10YWJsZV9fd3JhcHBlcj50YWJsZT50Ym9keT50cjpob3Zlcjpub3QoLnYtZGF0YS10YWJsZV9fZXhwYW5kZWRfX2NvbnRlbnQpOm5vdDEuMzcxIEUtLjAxNzA3CkcxIFgxNDguODMgWTEyMS40MDcgRS0uMDE2OTYKRzEgWDE0OS4xMzQgWTEyMS4xOTIgRS0uMDg1OTgKRzEgWDE0OS4zNTYgWTEyMS4xMTIgRS0uMDU0NDkKRzEgWDE0OS40NDggWTEyMS4wMDEgRS0uMDMzMjkKRzEgWDE0OS43ODQgWTEyMC45NzUgRS0uMDc3ODIKRzEgWDE0OS44NDcgWTEyMC44NjQgRS0uMDI5NDcKRzEgWDE1MC41NTMgWTEyMC41OTMgRS0uMTc0NjEKRzEgWDE1MC42NiBZMTIwLjU1OSBFLS4wMjU5MgpHMSBYMTUwLjc3MiBZMTIwLjQ1NyBFLS4wMzQ5OApHMSBYMTUxLjE2NSBZMTIwLjI3NCBFLS4xMDAxCkcxIFgxNTEuNDQ3IFkxMjAuMTY0IEUtLjA2OTg5CkcxIFgxNTEuNjk4IFkxMjAuMDA1IEUtLjA2ODYxCkcxIFgxNTEuNzQxIFkxMTkuOTg3IEUtLjAxMDgxCjtXSVBFX0VORApHMSBaMi4yNSBGNzIwCkcxIFgxNTEuOTIgWTExOS4zNDkgRjEwODAwCkcxIFoxLjg1IEY3MjAKRzEgRS44IEYyMTAwCk0yMDQgUzgwMApHMSBGMTUwMApHMSBYMTUxLjYgWTExOS40OTYgRS4wMDc0NwpHMSBYMTUxLjUxNSBZMTE5LjU3NSBFLjAwMjQ2CkcxIFgxNTEuMjIgWTExOS42MjkgRS4wMDYzNgpHMSBYMTUxLjEwNiBZMTE5Ljc1IEUuMDAzNTIKRzEgWDE1MC44MzkgWTExOS44NDYgRS4wMDYwMgpHMSBYMTUwLjU3MiBZMTE5Ljk4NyBFLjAwNjQKRzEgWDE1MC4zNTggWTEyMC4wNyBFLjAwNDg3CkcxIFgxNTAuMjk2IFkxMjAuMTQ2IEUuMDAyMDgKRzEgWDE0OS42NzUgWTEyMC4zOTQgRS4wMTQxOApHMSBYMTQ5LjQ0NiBZMTIwLjUzMyBFLjAwNTY4CkcxIFgxNDguOTU4IFkxMjAuNzc5IEUuMDExNTkKRzEgWDE0OC43MzYgWTEyMC44NDQgRS4wMDQ5CkcxIFgxNDguNjMgWTEyMC45NzEgRS4wMDM1MQpHMSBYMTQ4LjQwMyBZMTIwLjk4NSBFLjAwNDgyCkcxIFgxNDguNDAzIFkxMjEuMDU4IEUuMDAxNTUKRzEgWDE0Ny45NiBZMTIxLjI0OSBFLjAxMDIzCjtXSURUSDowLjQxODEzMgpHMSBYMTQ3Ljg1NCBZMTIxLjE4IEUuMDAzMDQKO1dJRFRIOjAuNDY0MDc0CkcxIFgxNDcuNzQ4IFkxMjEuMTEgRS4wMDM0Mgo7V0lEVEg6MC41MTAwMTYKRzEgWDE0Ny42NDIgWTEyMS4wNDEgRS4wMDM3Nwo7V0lEVEg6MC41NTU5NTgKRzEgWDE0Ny41MzYgWTEyMC45NzIgRS4wMDQxMwpHMSBYMTQ3LjQ3MSBZMTIxLjAyNiBFLjAwMjc2CjtXSURUSDowLjUxMDAxNgpHMSBYMTQ3LjQwNSBZMTIxLjA4MSBFLjAwMjU2CjtXSURUSDowLjQ2NDA3NApHMSBYMTQ3LjM0IFkxMjEuMTM1IEUuMDAyMjgKO1dJRFRIOjAuNDE4MTMyCkcxIFgxNDcuMjc1IFkxMjEuMTkgRS4wMDIwNQo7V0lEVEg6MC40MTQzMTIKRzEgWDE0Ny4zNDMgWTEyMS4xMjkgRS4wMDIxOAo7V0lEVEg6MC40NTY0MzMKRzEgWDE0Ny40MTEgWTEyMS4wNjcgRS4wMDI0Mwo7V0lEVEg6MC40OTg1NTUKRzEgWDE0Ny40OCBZMTIxLjAwNiBFLjAwMjY4CjtXSURUSDowLjU0MDY3NgpHMSBYMTQ3LjU0OCBZMTIwLjk0NSBFLjAwMjkKRzEgWDE0Ny41MjMgWTEyMC44ODQgRS4wMDIwOQo7V0lEVEg6MC40OTg1NTUKRzEgWDE0Ny40OTcgWTEyLjkzNCBFLjAwODYyCkcxIFgxMDYuNDggWTE4OC4yNTYgRS4wMTc2OApHMSBYMTA2LjU3OSBZMTg3Ljk2NyBFLjAwNzk2CkcxIFgxMDYuNzkxIFkxODcuNTc2IEUuMDExNTkKRzEgWDEwNi40MDcgWTE4Ny4xNTQgRS4wMTQ4NwpHMSBYMTA2LjE4NyBZMTg2LjY0NSBFLjAxNDQ1CkcxIFgxMDYuMTQ3IFkxODYuMzUzIEUuMDA3NjgKRzEgWDEwNi4wOTEgWTE4Ni4yOTYgRS4wMDIwOApHMSBYMTA2LjAwMyBZMTg2LjAwNyBFLjAwNzg3CkcxIFgxMDYuMDEgWTE4NS44MiBFLjAwNDg4CjtXSURUSDowLjQyMjI0NApHMSBYMTA2LjAxOCBZMTg1LjYzMyBFLjAwNDU1CjtXSURUSDowLjM5NDQ4OApHMSBYMTA2LjEwNSBZMTg1LjUxIEUuMDAzNAo7V0lEVEg6MC4zNjg5MjEKRzEgWDEwNi4xOTMgWTE4NS4zODcgRS4wMDMxOAo7V0lEVEg6MC4zNjM0MDUKRzEgWDEwNi41ODMgWTE4NS4wMzEgRS4wMTA5MQo7V0lEVEg6MC4zOTU0MDEKRzEgWDEwNi45NTkgWTE4NC42ODcgRS4wMTE1NAo7V0lEVEg6MC40MTQ0OQpHMSBYMTA3LjM5MSBZMTg0LjI0OCBFLjAxNDY4CjtXSURUSDowLjQ0NDk1MQpHMSBYMTA3Ljc1OSBZMTgzLjg3OCBFLjAxMzQzCjtXSURUSDowLjQ3NTk3OApHMSBYMTA4LjIyNyBZMTgzLjM1IEUuMDE5NTMKO1dJRFRIOjAuNTA0Njc1CkcxIFgxMDguNTk0IFkxODIuOTIzIEUuMDE2NTkKO1dJRFRIOjAuNTE3MDc1CkcxIFgxMDguODU4IFkxODIuNTY4IEUuMDEzMzgKRzEgWDEwOS4wMjcgWTE4Mi4yNTcgRS4wMTA3CjtXSURUSDowLjUxMjc0NgpHMSBYMTA5LjMyNiBZMTgxLjg0NCBFLjAxNTI4CjtXSURUSDowLjUyNDk5MwpHMSBYMTA5LjYwNiBZMTgxLjQ1MiBFLjAxNDgKRzEgWDEwOS42ODQgWTE4MS4yODQgRS4wMDU2OQo7V0lEVEg6MC40OTcyNTkKRzEgWDEwOS43NjIgWTE4MS4xMTYgRS4wMDUzNwo7V0lEVEg6MC40ODUwNzcKRzEgWDExMC4xNTcgWTE4MC41NjkgRS4wMTkwNgpHMSBYMTEwLjI1MSBZMTgwLjM0NSBFLjAwNjg2CjtXSURUSDowLjQ1NzM1OApHMSBYMTEwLjM0NSBZMTgwLjEyMiBFLjAwNjQyCjtXSURUSDowLjQ1MDExOApHMSBYMTEwLjYwNCBZMTc5LjcwMSBFLjAxMjg4CkcxIFgxMTAuNjYzIFkxNzkuNTI2IEUuMDA0ODEKO1dJRFRIOjAuNDIxNzMzCkcxIFgxMTAuNzIzIFkxNzkuMzUyIEUuMDA0NDcKO1dJRFRIOjAuNDA1MzkKRzEgWDExMC44NTggWTE3OS4xNTYgRS4wMDU1NAo7V0lEVEg6MC40NTM2MjMKRzEgWDExMC44NjIgWTE3OS4wNTMgRS4wMDI3MQo7V0lEVEg6MC41MDE4NTUKRzEgWDExMC44NjYgWTE3OC45NSBFLjAwMzAyCjtXSURUSDowLjU1MDA4NwpHMSBYMTEwLjg2OSBZMTc4Ljg0OCBFLjAwMzMKO1dJRFRIOjAuNTk4MzIKRzEgWDExMC44NzMgWTE3OC43NDUgRS4wMDM2NAo7V0lEVEg6MC42NDY1NTIKRzEgWDExMC44NzcgWTE3OC42NDIgRS4wMDM5NQpHMSBYMTEwLjgyMiBZMTc4LjY4OCBFLjAwMjc1CjtXSURUSDowLjYwMjIxOQpHMSBYMTEwLjc2OCBZMTc4LjczMyBFLjAwMjUKO1dJRFRIOjAuNTU3ODg1CkcxIFgxMTAuNzEzIFkxNzguNzc4IEUuMDAyMzMKO1cgWDc0Ljg4NiBZNzQuNTE2IEUuMDIzNQpHMSBYNzUuMDE2IFk3NS41ODkgRS4wMjgxNgpHMSBYNzUuMTYyIFk3Ni4wNTEgRS4wMTI2MgpHMSBYNzUuMjMzIFk3Ni42MDcgRS4wMTQ2CkcxIFg3NS4zODkgWTc3LjIwNCBFLjAxNjA4CkcxIFg3NS40OSBZNzcuODUyIEUuMDE3MDkKRzEgWDc1LjM2OCBZNzguMzE1IEUuMDEyNDgKRzEgWDc1LjMzMiBZNzguNzcgRS4wMTE4OQpHMSBYNzUuMzc3IFk3OS4yMTQgRS4wMTE2MwpHMSBYNzUuNTAyIFk3OS43MjIgRS4wMTM2MwpHMSBYNzUuNTczIFk4MC4yNDggRS4wMTM4MwpHMSBYNzUuNzAzIFk4MC44ODkgRS4wMTcwNApHMSBYNzUuODA2IFk4MS41OTYgRS4wMTg2MgpHMSBYNzUuOTI5IFk4MS44MjUgRS4wMDY3NwpHMSBYNzYuMDQxIFk4Mi4xOTIgRS4wMQpHMSBYNzYuMjA2IFk4Mi41NjcgRS4wMTA2NwpHMSBYNzYuNDUgWTgzLjAyNSBFLjAxMzUyCkcxIFg3Ni41NjQgWTgzLjA0IEUuMDAzCkcxIFg3Ni41NjIgWTgzLjI1MiBFLjAwNTUyCkcxIFg3Ni42NDYgWTgzLjM5MyBFLjAwNDI4CkcxIFg3Ni43ODcgWTgzLjQyOSBFLjAwMzc5CkcxIFg3Ni44NDYgWTgzLjc2MyBFLjAwODg0CkcxIFg3Ni44OTkgWTgzLjg5OCBFLjAwMzc4CkcxIFg3Ni45ODEgWTgzLjkzOCBFLjAwMjM4CkcxIFg3Ni45ODcgWTg0LjA0IEUuMDAyNjYKRzEgWDc3LjE4NSBZODQuMjk3IEUuMDA4NDUKRzEgWDc3LjIwNiBZODQuNjkyIEUuMDEwMzEKRzEgWDc3LjQwNSBZODQuODYgRS4wMDY3OQpHMSBYNzcuMzgxIFk4NS4xOTUgRS4wMDg3NQpHMSBYNzcuNDMzIFk4NS4zNDggRS4wMDQyMQpHMSBYNzcuNTg2IFk4NS40MTcgRS4wMDQzNwpHMSBYNzcuNjA3IFk4NS40OTIgRS4wMDIwMwpHMSBYNzcuNzc4IFk4NS42OSBFLjAwNjgyCkcxIFg3Ny43MzYgWTg1Ljg2NCBFLjAwNDY2CjtXSURUSDowLjQ3NDA4NQpHMSBYNzcuNjc2IFk4Ni4xMzggRS4wMDc3Mwo7V0lEVEg6MC41MTExNzQKRzEgWDc3LjY2OSBZODYuMjk2IEUuMDA0NzIKO1dJRFRIOjAuNTQ4MjYyCkcxIFg3Ny42NjIgWTg2LjQ1NCBFLjAwNTA5CkcxIFg3Ny44MTQgWTg2LjYzNCBFLjAwNzU4CjtXSURUSDowLjUzMzk1NQpHMSBYNzcuNzgxIFk4Ni43MTggRS4wMDI4MgpHMSBYNzcuODE5IFk4Ni44OTEgRS4wMDU1NApHMSBYNzcuOTM3IFk4Ni45NzkgRS4wMDQ2MQpHMSBYNzcuODYxIFk4Ny4wNDYgRS4wMDMxNwpHMSBYNzcuNzc3IFk4Ny4yMDcgRS4wMDU2OApHMSBYNzcuNzg0IFk4Ny41NjYgRS4wMTEyNApHMSBYNzcuNjM5IFk4Ny4yODQgRS4wMDk5MgpHMSBYNzcuNTExIFk4Ny4xMDIgRS4wMDY5Ngo7V0lEVEg6MC41NDgyNjIKRzEgWDc3LjE4MSBZODYuNjc0IEUuMDE3MzkKRzEgWDc3LjE1OCBZODYuNDggRS4wMDYyOQo7V0lEVEg6MC41MzcwNzIKRzEgWDc3LjA2OSBZODYuMjU0IEUuMDA3NjUKO1dJRFRIOjAuNDkzNTM2CkcxIFg3Ni45OCBZODYuMDI4IEUuMDA2OTkKO1dJRFRIOjAuNDQ5OTk5CkcxIFg3Ni44NDYgWTg1Ljk0MiBFLjAwNDE1CkcxIFg3Ni43OTMgWTg1LjgzIEUuMDAzMjMKRzEgWDc2LjY2OSBZODUuNzM1IDY1NApHMSBYMTQ1Ljk1MyBZMTQ4LjEyMyBFLjAxOTQxCjtXSURUSDowLjM3MjE5CkcxIFgxNDYuMzkgWTE0Ny42NDggRS4wMTM2OQo7V0lQRV9TVEFSVApHMSBGODY0MApHMSBYMTQ1Ljk1MyBZMTQ4LjEyMyBFLS4xNDkwMwpHMSBYMTQ1LjIzNyBZMTQ4LjMyOCBFLS4xNzE5NwpHMSBYMTQ2LjUgWTE0Ny4wNjUgRS0uNDEyNDMKRzEgWDE0Ni41MDggWTE0Ni43NzcgRS0uMDY2NTcKO1dJUEVfRU5ECkcxIFoxLjk1IEY3MjAKRzEgWDEzMy41OTMgWTE1MC40NTYgRjEwODAwCkcxIFoxLjU1IEY3MjAKRzEgRS44IEYyMTAwCjtUWVBFOlRvcCBzb2xpZCBpbmZpbGwKO1dJRFRIOjAuNDAwNDg5CkcxIEYyNDAwCkcxIFgxMzIuOTE2IFkxNTEuMTMzIEUuMDIxOTkKRzEgWDEzMi44MDkgWTE1MC43MTkgRjEwODAwCkcxIEYyNDAwCkcxIFgxMzMuNjQ2IFkxNDkuODgyIEUuMDI3MTkKRzEgWDEzMy44NCBZMTQ5LjE2OCBGMTA4MDAKRzEgRjI0MDAKRzEgWDEzMi42MiBZMTUwLjM4OCBFLjAzOTYzCkcxIFgxMzIuMzIgWTE1MC4xNjcgRjEwODAwCkcxIEYyNDAwCkcxIFgxMzMuOTA1IFkxNDguNTgyIEUuMDUxNDgKRzEgWDEzMy44NzkgWTE0OC4wODcgRjEwODAwCkcxIEYyNDAwCkcxIFgxMzEuODQzIFkxNTAuMTIzIEUuMDY2MTMKRzEgWDEzMS4yOTkgWTE1MC4xNDYgRjEwODAwCkcxIEYyNDAwCkcxIFgxMzMuNzQxIFkxNDcuNzA0IEUuMDc5MzIKRzEgWDEzMy42MTYgWTE0Ny4zMDggRjEwODAwCkcxIEYyNDAwCkcxIFgxMzAuOTc1IFkxNDkuOTQ5IEUuMDg1NzgKRzEgWDEzMC42OTQgWTE0OS43MSBGMTA4MDAKRzEgRjI0MDAKRzEgWDEzMy41MTEgWTE0Ni44OTIgRS4wOTE1MgpHMSBYMTMzLjM1NCBZMTQ2LjUyOCBGMTA4MDAKRzEgRjI0MDAKRzEgWDEzMC40MDQgWTE0OS40NzkgRS4wOTU4NApHMSBYMTMwLjEyMiBZMTQ5LjIzOSBGMTA4MDAKRzEgRjI0MDAKRzEgWDEzMy4xNDQgWTE0Ni4yMTggRS4wOTgxNApHMSBYMTMyLjk1MyBZMTQ1Ljg4OCBGMTA4MDAKRzEgRjI0MDAKRzEgWDEyOS44NDkgWTE0OC45OTIgRS4xMDA4MgpHMSBYMTI5LjU5OSBZMTQ4LjcyMSBGMTA4MDAKRzEgRjI0MDAKRzEgWDEzMi43NDQgWTE0NS41NzYgRS4xMDIxNgpHMSBYMTMyLjUyMyBZMTQ1LjI3NiBGMTA4MDAKRzEgRjI0MDAKRzEgWDEyOS4zNzggWTE0OC40MjIgRS4xMDIxNwpHMSBYMTI5LjE1OCBZMTQ4LjEyIEYxMDgwMApHMSBGMjQwMApHMSBYMTMyLjI5MyBZMTQ0Ljk4NSBFLjEwMTgzCkcxIFgxMzIuMDk0IFkxNDQuNjY0IEYxMDgwMApHMSBGMjQwMApHMSBYMTI4Ljk0NyBZMTQ3LjgxMSBFLjEwMjIyCkcxIFgxMjguNzcyIFkxNDcuNDY1IEYxMDgwMApHMSBGMjQwMApHMSBYMTMxLjg1NiBZMTQ0LjM4IEUuMTAwMTkKRzEgWDEzMS42NiBZMTQ0LjA1NiBGMTA4MDAKRzEgRjI0MDAKRzEgWDEyOC41OTcgWTE0Ny4xMTggRS4wOTk0OApHMSBYMTI4LjQyMiBZMTQ2Ljc3MyBGMTA4MDAKRzEgRjI0MDAKRzEgWDEzMS40NTcgWTE0My43MzggRS4wOTg1OAo7V0lQRV9TVEFSVApHMSBGODY0MApHMSBYMTI5LjAwNyBZMTQ2LjE4OCBFLS44IFgxODIuMTE1IFk2NC42NTIgRS4wMTY2MwpHMSBYMTgyLjEyOSBZNjQuMzc0IEUuMDA3MjUKRzEgWDE4Mi4yMSBZNjQuMDg0IEUuMDA3ODUKRzEgWDE4MS44ODYgWTYzLjY3OCBFLjAxMzUzCkcxIFgxODEuNTkgWTYzLjIyNiBFLjAxNDA4CkcxIFgxODEuNDIzIFk2Mi44NzIgRS4wMTAyCkcxIFgxODEuMzk5IFk2Mi43OCBFLjAwMjQ4CkcxIFgxODAuODA3IFk2Mi40ODcgRS4wMTcyMQpHMSBYMTgwLjczOSBZNjIuNDM3IEUuMDAyMgpHMSBYMTgwLjc0NCBZNjIuOTI1IEUuMDEyNzIKRzEgWDE4MC40NjQgWTYzLjM2NCBFLjAxMzU3CkcxIFgxODAuMDQxIFk2My43ODggRS4wMTU2MQpHMSBYMTc5Ljk3NyBZNjMuODMxIEUuMDAyMDEKRzEgWDE4MC4yODYgWTY0LjI3MSBFLjAxNDAxCkcxIFgxODAuNDUzIFk2NC42MDQgRS4wMDk3MQpHMSBYMTgwLjYwMiBZNjUuMTE2IEUuMDEzODkKRzEgWDE4MC42NDcgWTY1LjE2NSBFLjAwMTczCkcxIFgxODEuMTkxIFk2NS4zOTMgRS4wMTUzNwpHMSBYMTgxLjQwOCBZNjUuNTIzIEUuMDA2NTkKRzEgWDE4MS41ODggWTY1LjcgRS4wMDY1OAo7V0lEVEg6MC40MTQzNDkKRzEgWDE4MS43NjcgWTY1Ljg3NyBFLjAwNgo7V0lEVEg6MC4zNzg2OTkKRzEgWDE4Mi4xMzcgWTY2LjM0IEUuMDEyODEKO1dJRFRIOjAuMzg0NDcxCkcxIFgxODIuMzE0IFk2Ni41NzEgRS4wMDYzOQo7V0lEVEg6MC40MTEyNTcKRzEgWDE4Mi40OSBZNjYuODAyIEUuMDA2ODcKO1dJRFRIOjAuNDQ5OTk5CkcxIFgxODIuODM1IFk2Ny4yNzYgRS4wMTUyOApHMSBYMTgzLjExIFk2Ny43NzEgRS4wMTQ3NQpHMSBYMTgzLjI3IFk2OC4xNDEgRS4wMTA1CkcxIFgxODMuNDAxIFk2OC43NTkgRS4wMTY0NgpHMSBYMTgzLjQgWTY5LjI4NCBFLjAxMzY4CkcxIFgxODMuMzgzIFk2OS40NCBFLjAwNDA5CkcxIFgxODMuODU3IFk2OS41NjkgRS4wMTI4CkcxIFgxODQuMzE0IFk2OS43ODMgRS4wMTMxNQo7V0lEVEg6MC40NTI4MDMKRzEgWDE4NC43MjUgWTcwLjA5IEUuMDEzNDYKO1dJRFRIOjAuNDcwMzU0CkcxIFgxODUuMTExIFk3MC41MjggRS4wMTU5NQo7V0lEVEg6MC41MTAxMDIKRzEgWDE4NS40NDkgWTcwLjk4NSBFLjAxNjk0CkcxIFgxODUuODYxIFk3MC41NzIgRS4wMTczOQpHMSBYMTg1LjU3MSBZNzAuMzYyIEUuMDEwNjcKO1dJRFRIOjAuNDk2ODQ1CkcxIFgxODUuMjE5IFk2OS45OTEgRS4wMTQ4Mgo7V0lEVEg6MC40NTAyNDMKRzEgWDE4NC45MjUgWTY5LjU1OCBFLjAxMzY0CjtXSURUSDowLjQ0OTk5OQpHMSBYMTg0LjYxNSBZNjkuMTE2IEUuMDE0MDcKRzEgWDE4NC4zNjcgWTY4LjU3MyBFLjAxNTU1CkcxIFgxODQuMTg3IFk2OC4xMDUgRS4wMTMwNgpHMSBYMTg0LjA4NyBZNjcuNTU5IEUuMDE0NDYKRzEgWDE4My44NjkgWTY3LjEyNiBFLjAxMjYzCkcxIFgxODMuMjcgWTY2Ljk0MiBFLjAxNjMzCkcxIFgxODIuODA5IFk2Ni42MTYgRS4wMTQ3MQo7V0lEVEg6MC40MjczNjgKRzEgWDE4Mi42MzMgWTY2LjQxNiBFLjAwNjU3CjtXSURUSDowLjM5Mzg0NApHMSBYMTgyLjQ1NiBZNjYuMjE2IEUuMDA2MDIKO1c1Ny40NzkgWTQ4LjI3NiBFLjAwNTA2CjtXSURUSDowLjUzMDcyNQpHMSBYMTU3LjMwOSBZNDguMzI1IEUuMDA1NQo7V0lEVEg6MC41NjUzNTMKRzEgWDE1Ny4yMjIgWTQ4LjMyNiBFLjAwMjg5CjtXSURUSDowLjU5OTk4CkcxIFgxNTcuMTM1IFk0OC4zMjcgRS4wMDMwOApHMSBYMTU3LjE5NSBZNDguMzkgRS4wMDMwOAo7V0lEVEg6MC41NjUzNTMKRzEgWDE1Ny4yNTYgWTQ4LjQ1NCBFLjAwMjk0CjtXSURUSDowLjUzMDcyNQpHMSBYMTU3LjMzNiBZNDguNTc3IEUuMDA0NTYKO1dJRFRIOjAuNDkwMzYyCkcxIFgxNTcuNDE3IFk0OC43IEUuMDA0MjEKO1dJRFRIOjAuNDQ5OTk5CkcxIFgxNTcuNzcxIFk0OS4wNjQgRS4wMTMyMwpHMSBYMTU3LjgzNyBZNDkuMjA4IEUuMDA0MTMKRzEgWDE1OC4zNTcgWTQ5LjQ3NyBFLjAxNTI1CkcxIFgxNTguNjM5IFk0OS44NzMgRS4wMTI2NwpHMSBYMTU5LjAwOCBZNTAuMjU5IEUuMDEzOTEKRzEgWDE1OS4zNDIgWTUwLjY2NiBFLjAxMzcyCkcxIFgxNTkuNjgxIFk1MS4xNzggRS4wMTYKRzEgWDE2MC4wOTUgWTUxLjYyNSBFLjAxNTg3CkcxIFgxNjAuMjkgWTUxLjg3MiBFLjAwODIKRzEgWDE2MC4zMDMgWTUxLjkzMSBFLjAwMTU3CkcxIFgxNjAuNTc1IFk1Mi4yNDggRS4wMTA4OApHMSBYMTYwLjcxOSBZNTIuNSBFLjAwNzU2CkcxIFgxNjEuMTE5IFk1My4wNSBFLjAxNzcyCkcxIFgxNjEuMzc3IFk1My40NzkgRS4wMTMwNApHMSBYMTYxLjU4NCBZNTMuNzUgRS4wMDg4OQpHMSBYMTYyLjM4MyBZNTUuMzIzIEUuMDQ1OTcKRzEgWDE2Mi42MDkgWTU1LjcxNiBFLjAxMTgxCkcxIFgxNjIuODczIFk1Ni4zNjggRS4wMTgzMwpHMSBYMTYzLjYzNyBZNTguNjExIEUuMDYxNzQKRzEgWDE2My45NiBZNTkuNjY1IEUuMDI4NzIKRzEgWDE2NC4wMTggWTYwLjE5OSBFLjAxNApHMSBYMTY0LjI2MyBZNjEuNDY4IEUuMDMzNjgKRzEgWDE2NC4zNTIgWTYyLjIxIEUuMDE5NDcKRzEgWDE2NC4zODkgWTYyLjc1NyBFLjAxNDI5CkcxIFgxNjQuNDIzIFk2NC43OSBFLjA1Mjk4CkcxIFgxNjQuNDA1IFk2Ni4xNzIgRS4wMzYwMQpHMSBYMTY0LjM2NyBZNjYuOTU5IEUuMDIwNTMKRzEgWDE2NC4xMTQgWTY4LjU4NCBFLjA0Mjg1CkcxIFgxNjQuMDM3IFk2OC44MTMgRS4wMDYzCkcxIFgxNjQuMDEgWTY5LjQwNCBFLjAxNTQxCkcxIFgxNjMuOTAzIFk2OS45IEUuMDEzMjIKRzEgWDE2My43MyBZNzAuMzg2IEUuMDEzNDQKRzEgWDE2My42MjggWTcwLjYxMSBFLjAwNjQ0CkcxIFgxNjMuNTUxIFk3MS40MDcgRS4wMjA4NApHMSBYMTYzLjQxOCBZNzEuNzk5IEUuMDEwNzkKRzEgWDE2My4yNDIgWTcyLjQ4OSBFLjAxODU1CkcxIFgxNjMuMjYyIFk3My4wMjMgRS4wMTM5MgpHMSBYMTYzLjAyMiBZNzMuOTYyIEUuMDI1MjUKRzEgWDE2Mi44MjYgWTc0LjQzNiBFLjAxMzM2CkcxIFgxNjIuMzUxIFk3NS40MjcgRS4wMjg2MwpHMSBYMTYyLjA5OSBZNzUuODg5IEUuMDEzNzEKRzEgWDE2MS42MjIgWTc2LjkyNiBFLjAyOTc0CkcxIFgxNjEuMzggWTc3LjM3NCBFLjAxMzI3CkcxIFgxNjEuMTUgWTc3LjkyMSBFLnsiYWN0dWFsIjoyOC44LCJ0YXJnZXQiOm51bGx9fV0sImJ1c3lGaWxlcyI6W10sIm1hcmtpbmdzIjpbeyJ0eXBlIjoiY29ubmVjdGVkIiwibGFiZWwiOiJDb25uZWN0ZWQiLCJ0aW1lIjoxNzE3MzAyNDE1LjU1MzY4Njl9XSwibG9ncyI6WyJSZWN2OiBUOjIwLjUgLzAuMCBCOjE4LjkgLzAuMCBUMDoyMC41IC8wLjAgQDowIEJAOjAgUDowLjAgQToyOC44Il0sIm1lc3NhZ2VzIjpbIlQ6MjAuNSAvMC4wIEI6MTguOSAvMC4wIFQwOjIwLjUgLzAuMCBAOjAgQkA6MCBQOjAuMCBBOjI4LjgiLCIiXX19XTAKRzEgRjI0MDAKRzEgWDE2OC4xNjIgWTE2NS43OTggRS4xMTg5NgpHMSBYMTY3Ljk4NCBZMTY2LjE1MSBGMTA4MDAKRzEgRjI0MDAKRzEgWDE3MS4zOTQgWTE2OS41NjEgRS4xMTI5CkcxIFgxNzEuMDUyIFkxNjkuNzUgRjEwODAwCkcxIEYyNDAwCkcxIFgxNjcuOTA0IFkxNjYuNjAyIEUuMTA0MjMKRzEgWDE2OC4wMTMgWTE2Ny4yNDIgRjEwODAwCkcxIEYyNDAwCkcxIFgxNzAuNzI0IFkxNjkuOTUyIEUuMDg5NzQKRzEgWDE3MC4zOTMgWTE3MC4xNTMgRjEwODAwCkcxIEYyNDAwCkcxIFgxNjguMDU3IFkxNjcuODE3IEUuMDc3MzQKRzEgWDE2Ny43MTcgWTE2OC4wMDggRjEwODAwCkcxIEYyNDAwCkcxIFgxNzAuMDM3IFkxNzAuMzI3IEUuMDc2OApHMSBYMTY5LjY5MSBZMTcwLjUxMyBGMTA4MDAKRzEgRjI0MDAKRzEgWDE2Ny40MSBZMTY4LjIzMiBFLjA3NTUyCkcxIFgxNjcuMTUgWTE2OC41MDMgRjEwODAwCkcxIEYyNDAwCkcxIFgxNjkuMzI5IFkxNzAuNjgyIEUuMDcyMTUKRzEgWDE2OC45NTMgWTE3MC44MzYgRjEwODAwCkcxIEYyNDAwCkcxIFgxNjYuOTcyIFkxNjguODU1IEUuMDY1NTkKRzEgWDE2Ni44NyBZMTY5LjI4NCBGMTA4MDAKRzEgRjI0MDAKRzEgWDE2OC41MjYgWTE3MC45NCBFLjA1NDgzCkcxIFgxNjcuOTIxIFkxNzAuODY2IEYxMDgwMApHMSBGMjQwMApHMSBYMTY2LjkzOSBZMTY5Ljg4NCBFLjAzMjUxCjtXSVBFX1NUQVJUCkcxIEY4NjQwCkcxIFgxNjcuOTIxIFkxNzAuODY2IEUtLjMyMDY3CjtXSVBFX0VORApHMSBFLS40NzkzMyBGMjEwMApHMSBaMS41IEY3MjAKRzEgWDE3NC4wNjcgWTE3My42OTIgRjEwODAwCkcxIFoxLjEgRjcyMApHMSBFLjggRjIxMDAKO1RZUEU6SW50ZXJuYWwgaW5maWxsCjtXSURUSDowLjQ1CkcxIEY0ODAwCkcxIFgxNzMuOTczIFkxNzMuMzcxIEUuMDA4NzIKRzEgWDE3My44NjkgWTE3My4xOCBFLjAwNTY3CkcxIFgxNzMuNzYxIFkxNzMuMDU4IEUuMDA0MjUKRzEgWDE3NC4wNjcgWTE3Mi44NTggRS4wMDk1MwpHMSBYMTc0LjMgWTE3Mi43NDUgRS4wMDY3NQpHMSBYMTc0LjQyMyBZMTcyLjY1MiBFLjAwNDAyCkcxIFgxNzQuNDU1IFkxNzIuNjY0IEUuMDAwODkKRzEgWDE3NC42NzQgWTE3Mi42MTYgRS4wMDU4NApHMSBYMTc0LjkzMyBZMTcyLjU0MyBFLjAwNzAxCkcxIFgxNzUuMDc0IFkxNzIuNDY4IEUuMDA0MTYKRzEgWDE3NS4zODEgWTE3Mi40MDEgRS4wMDgxOQpHMSBYMTc2LjEwNSBZMTcyLjU5NCBFLjAxOTUyCkcxIFgxNzYuNTUzIFkxNzIuNTUzIEUuLmtleXMoT2JqZWN0LmFzc2lnbih7fSxlLHMpKSxmdW5jdGlvbih1KXtjb25zdCBkPWxbdV18fGksbT1kKGVbdV0sc1t1XSx1KTtLLmlzVW5kZWZpbmVkKG0pJiZkIT09b3x8KHRbdV09bSl9KSx0fWNvbnN0IE0kPSIxLjYuOCIsTWY9e307WyJvYmplY3QiLCJib29sZWFuIiwibnVtYmVyIiwiZnVuY3Rpb24iLCJzdHJpbmciLCJzeW1ib2wiXS5mb3JFYWNoKChlLHMpPT57TWZbZV09ZnVuY3Rpb24ocil7cmV0dXJuIHR5cGVvZiByPT09ZXx8ImEiKyhzPDE/Im4gIjoiICIpK2V9fSk7Y29uc3QgVl89e307TWYudHJhbnNpdGlvbmFsPWZ1bmN0aW9uKHMsdCxyKXtmdW5jdGlvbiBpKG4sYSl7cmV0dXJuIltBeGlvcyB2IitNJCsiXSBUcmFuc2l0aW9uYWwgb3B0aW9uICciK24rIiciK2ErKHI/Ii4gIityOiIiKX1yZXR1cm4obixhLG8pPT57aWYocz09PSExKXRocm93IG5ldyBNZShpKGEsIiBoYXMgYmVlbiByZW1vdmVkIisodD8iIGluICIrdDoiIikpLE1lLkVSUl9ERVBSRUNBVEVEKTtyZXR1cm4gdCYmIVZfW2FdJiYoVl9bYV09ITAsY29uc29sZS53YXJuKGkoYSwiIGhhcyBiZWVuIGRlcHJlY2F0ZWQgc2luY2UgdiIrdCsiIGFuZCB3aWxsIGJlIHJlbW92ZWQgaW4gdGhlIG5lYXIgZnV0dXJlIikpKSxzP3MobixhLG8pOiEwfX07ZnVuY3Rpb24gdEcoZSxzLHQpe2lmKHR5cGVvZiBlIT0ib2JqZWN0Iil0aHJvdyBuZXcgTWUoIm9wdGlvbnMgbXVzdCBiZSBhbiBvYmplY3QiLE1lLkVSUl9CQURfT1BUSU9OX1ZBTFVFKTtjb25zdCByPU9iamVjdC5rZXlzKGUpO2xldCBpPXIubGVuZ3RoO2Zvcig7aS0tID4wOyl7Y29uc3Qgbj1yW2ldLGE9c1tuXTtpZihhKXtjb25zdCBvPWVbbl0sbD1vPT09dm9pZCAwfHxhKG8sbixlKTtpZihsIT09ITApdGhyb3cgbmV3IE1lKCJvcHRpb24gIituKyIgbXVzdCBiZSAiK2wsTWUuRVJSX0JBRF9PUFRJT05fVkFMVUUpO2NvbnRpbnVlfWlmKHQhPT0hMCl0aHJvdyBuZXcgTWUoIlVua25vd24gb3B0aW9uICIrbixNZS5FUlJfQkFEX09QVElPTil9fWNvbnN0IFltPXthc3NlcnRPcHRpb25zOnRHLHZhbGlkYXRvcnM6TWZ9LHFzPVltLnZhbGlkYXRvcnM7Y2xhc3MgQWx7Y29uc3RydWN0b3Iocyl7dGhpcy5kZWZhdWx0cz1zLHRoaXMuaW50ZXJjZXB0b3JzPXtyZXF1ZXN0Om5ldyBrXyxyZXNwb25zZTpuZXcga199fWFzeW5jIHJlcXVlc3Qocyx0KXt0cnl7cmV0dXJuIGF3YWl0IHRoaXMuX3JlcXVlc3Qocyx0KX1jYXRjaChyKXtpZihyIGluc3RhbmNlb2YgRXJyb3Ipe2xldCBpO0Vycm9yLmNhcHR1cmVTdGFja1RyYWNlP0Vycm9yLmNhcHR1cmVTdGFja1RyYWNlKGk9e30pOmk9bmV3IEVycm9yO2NvbnN0IG49aS5zdGFjaz9pLnN0YWNrLnJlcGxhY2UoL14uK1xuLywiIik6IiI7ci5zdGFjaz9uJiYhU3RyaW5nKHIuc3RhY2spLmVuZHNXaXRoKG4ucmVwbGFjZSgvXi4rXG4uK1xuLywiIikpJiYoci5zdGFjays9IlxuIituKTpyLnN0YWNrPW59dGhyb3cgcn19X3JlcXVlc3Qocyx0KXt0eXBlb2Ygcz09InN0cmluZyI/KHQ9dHx8e30sdC51cmw9cyk6dD1zfHx7fSx0PWppKHRoaXNvcGVydHkob2JqZWN0LG9sZFZhcix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIE9jdG9QcmludENsaWVudC5kZXByZWNhdGVkKG9sZE5hbWVzcGFjZSsiLiIrb2xkVmFyLG5ld05hbWVzcGFjZSsiLiIrbmV3VmFyLGdldHRlcikoKTt9LHNldDpmdW5jdGlvbih2YWwpe09jdG9QcmludENsaWVudC5kZXByZWNhdGVkKG9sZE5hbWVzcGFjZSsiLiIrb2xkVmFyLG5ld05hbWVzcGFjZSsiLiIrbmV3VmFyLHNldHRlcikodmFsKTt9fSk7fTtPY3RvUHJpbnRDbGllbnQuZXNjYXBlUGF0aD1mdW5jdGlvbihwYXRoKXtyZXR1cm4gXy5tYXAocGF0aC5zcGxpdCgiLyIpLGZ1bmN0aW9uKHApe3JldHVybiBlbmNvZGVVUklDb21wb25lbnQocCk7fSkuam9pbigiLyIpO307cmV0dXJuIE9jdG9QcmludENsaWVudDt9KTsKOwoKLy8gc291cmNlOiBqcy9hcHAvY2xpZW50L3NvY2tldC5qcwooZnVuY3Rpb24oZ2xvYmFsLGZhY3Rvcnkpe2lmKHR5cGVvZiBkZWZpbmU9PT0iZnVuY3Rpb24iJiZkZWZpbmUuYW1kKXtkZWZpbmUoWyJPY3RvUHJpbnRDbGllbnQiLCJqcXVlcnkiLCJsb2Rhc2giLCJzb2NranMiXSxmYWN0b3J5KTt9ZWxzZXtmYWN0b3J5KGdsb2JhbC5PY3RvUHJpbnRDbGllbnQsZ2xvYmFsLiQsZ2xvYmFsLl8sZ2xvYmFsLlNvY2tKUyk7fX0pKHRoaXMsZnVuY3Rpb24oT2N0b1ByaW50Q2xpZW50LCQsXyxTb2NrSlMpe3ZhciBub3JtYWxDbG9zZT0xMDAwO3ZhciBPY3RvUHJpbnRTb2NrZXRDbGllbnQ9ZnVuY3Rpb24oYmFzZSl7dmFyIHNlbGY9dGhpczt0aGlzLmJhc2U9YmFzZTt0aGlzLm9wdGlvbnM9e3RpbWVvdXRzOlswLDEsMSwyLDMsNSw4LDEzLDIwLDQwLDEwMF0sY29ubmVjdFRpbWVvdXQ6NTAwMCx0cmFuc3BvcnRUaW1lb3V0OjQwMDAscmF0ZVNsaWRpbmdXaW5kb3dTaXplOjIwfTt0aGlzLnNvY2tldD11bmRlZmluZWQ7dGhpcy5yZWNvbm5lY3Rpbmc9ZmFsc2U7dGhpcy5yZWNvbm5lY3RUcmlhbD0wO3RoaXMucmVnaXN0ZXJlZEhhbmRsZXJzPXt9O3RoaXMucmF0ZVRocm90dGxlRmFjdG9yPTE7dGhpcy5yYXRlQmFzZT01MDA7dGhpcy5yYXRlTGFzdE1lYXN1cmVtZW50cz1bXTt0aGlzLmNvbm5lY3RUaW1lb3V0PXVuZGVmaW5lZDt0aGlzLm9uTWVzc2FnZSgiY29ubmVjdGVkIixmdW5jdGlvbigpe2lmKHNlbGYuY29ubmVjdFRpbWVvdXQpe2NsZWFyVGltZW91dChzZWxmLmNvbm5lY3RUaW1lb3V0KTtzZWxmLmNvbm5lY3RUaW1lb3V0PXVuZGVmaW5lZDt9fSk7fTtPY3RvUHJpbnRTb2NrZXRDbGllbnQucHJvdG90eXBlLnByb3BhZ2F0ZU1lc3NhZ2U9ZnVuY3Rpb24oZXZlbnQsZGF0YSl7dmFyIHN0YXJ0PW5ldyBEYXRlKCkuZ2V0VGltZSgpO3ZhciBldmVudE9iaj17ZXZlbnQ6ZXZlbnQsZGF0YTpkYXRhfTt2YXIgY2F0Y2hBbGxIYW5kbGVycz10aGlzLnJlZ2lzdGVyZWRIYW5kbGVyc1siKiJdO2lmKGNhdGNoQWxsSGFuZGxlcnMmJmNhdGNoQWxsSGFuZGxlcnMubGVuZ3RoKXtfLmVhY2goY2F0Y2hBbGxIYW5kbGVycyxmdW5jdGlvbihoYW5kbGVyKXtoYW5kbGVyKGV2ZW50T2JqKTt9KTt9CnZhciBoYW5kbG4mJmdsZShlLHQsbiksbn07bGV0IHFfPWNsYXNzIGV4dGVuZHMgRyhZKXtnZXQgZW5kc3RvcHMoKXtyZXR1cm4gdGhpcy4kc3RvcmUuZ2V0dGVyc1sicHJpbnRlci9nZXRFbmRzdG9wcyJdfWdldCBwcm9iZSgpe3JldHVybiB0aGlzLiRzdG9yZS5nZXR0ZXJzWyJwcmludGVyL2dldFByb2JlIl19Z2V0IGVuZHN0b3BzQW5kUHJvYmVzKCl7Y29uc3QgZT1bLi4udGhpcy5lbmRzdG9wc10sdD10aGlzLnByb2JlO3JldHVybiB0IT09dm9pZCAwJiZlLnB1c2goe25hbWU6IlByb2JlIixzdGF0ZTp0Lmxhc3RfcXVlcnk/InRyaWdnZXJlZCI6Im9wZW4ifSksZX1nZXQgaGFzRW5kc3RvcHMoKXtyZXR1cm4gdGhpcy5lbmRzdG9wcy5sZW5ndGg+MH1xdWVyeUVuZHN0b3BzKCl7Ui5wcmludGVyUXVlcnlFbmRzdG9wcygpLHRoaXMucHJvYmUhPT12b2lkIDAmJnRoaXMuc2VuZEdjb2RlKCJRVUVSWV9QUk9CRSIsdGhpcy4kd2FpdHMub25RdWVyeVByb2JlKX1kZXN0cm95ZWQoKXt0aGlzLiRzdG9yZS5jb21taXQoInByaW50ZXIvc2V0Q2xlYXJFbmRTdG9wcyIpfX07cV89X2xlKFtQKHtjb21wb25lbnRzOnt9fSldLHFfKTt2YXIgYmxlPWZ1bmN0aW9uKCl7dmFyIGU9dGhpcyx0PWUuX3NlbGYuX2M7cmV0dXJuIGUuX3NlbGYuX3NldHVwUHJveHksdChRZSx7YXR0cnM6e3RpdGxlOmUuJHQoImFwcC5nZW5lcmFsLnRpdGxlLmVuZHN0b3BzIiksInN1Yi10aXRsZSI6ZS4kdCgiYXBwLmVuZHN0b3AubXNnLnN1YnRpdGxlIiksaWNvbjoiJGV4cGFuZEhvcml6b250YWwifSxzY29wZWRTbG90czplLl91KFt7a2V5OiJjb2xsYXBzZS1idXR0b24iLGZuOmZ1bmN0aW9uKCl7cmV0dXJuW3Qoayx7c3RhdGljQ2xhc3M6Im1zLTEgbXktMSIsYXR0cnM6e2xvYWRpbmc6ZS5oYXNXYWl0KGUuJHdhaXRzLm9uUXVlcnlFbmRzdG9wcyl8fGUuaGFzV2FpdChlLiR3YWl0cy5vblF1ZXJ5UHJvYmUpLGNvbG9yOiIiLGZhYjoiIiwieC1zbWFsbCI6IiIsdGV4dDoiIn0sb246e2NsaWNrOmUucXVlcnlFbmRzdG9wc319LFt0KHgsW2UuX3YoIiRyZWZyZXNoIildKV0sMSldfSxwcm94eTohMH1dKX0sW2UuaGFzRW5kc3RvcHM/dChxdCxbdCgidGJvZHkiLGUuX2woZS5lbmRzdG9wc0FuZFByb2JlcyxmdW5jdGlvbihzKXtyZXR1cm4gdCgidHIiLHtrZXk6cy5uYW1lfSxbdCgidGQiLFt0KCJzcGFuIix7c3RhdGljQ2xhc3M6ImZvY3VzLS10ZXh0In0sW2UuX3YoZS5fcyhzLm5hbWUpKV0pXSksdCgidGQiLFt0KG1yLHthdHRyczp7Y29sb3I6cy5zdGF0ZT09PSJvcGVuIj8ic2Vjb25kYXJ5Ijoid2FybmluZyIsc21hbGw6IiIsbGFiZWw6IiJ9fSxbdCh4LHthdHRyczp7c21hbGw6IiIsbGVmdDoiIn19LFtlLl92KCIgIitlLl9zKHMuc3RhdGU9PT0ib3BlbiI/IiRibGFua0NpcmNsZSI6IiRtYXJrZWRDaXJjbGUiKSsiICIpXSksZS5fdigiICIrZS5fcyhlLiR0KCJhcHAuZW5kc3RvcC5sYWJlbC4iK3Muc3RhdGUudG9Mb3dlckNhc2UoKSkpKyIgIildLDEpXSwxKV0pfSksMCldKTplLl9lKCldLDEpfSx5bGU9W10sJGxlPUUocV8sYmxlLHlsZSwhMSxudGhpcy5tYXhXaWR0aCl8fCIwIn0sY2FsY3VsYXRlZE1pbldpZHRoKCl7aWYodGhpcy5taW5XaWR0aClyZXR1cm4gbWUodGhpcy5taW5XaWR0aCl8fCIwIjtjb25zdCByPU1hdGgubWluKHRoaXMuZGltZW5zaW9ucy5hY3RpdmF0b3Iud2lkdGgrTnVtYmVyKHRoaXMubnVkZ2VXaWR0aCkrKHRoaXMuYXV0bz8xNjowKSxNYXRoLm1heCh0aGlzLnBhZ2VXaWR0aC0yNCwwKSksZT1pc05hTihwYXJzZUludCh0aGlzLmNhbGN1bGF0ZWRNYXhXaWR0aCkpP3I6cGFyc2VJbnQodGhpcy5jYWxjdWxhdGVkTWF4V2lkdGgpO3JldHVybiBtZShNYXRoLm1pbihlLHIpKXx8IjAifSxjYWxjdWxhdGVkVG9wKCl7cmV0dXJuKHRoaXMuYXV0bz9tZSh0aGlzLmNhbGNZT3ZlcmZsb3codGhpcy5jYWxjdWxhdGVkVG9wQXV0bykpOnRoaXMuY2FsY1RvcCgpKXx8IjAifSxoYXNDbGlja2FibGVUaWxlcygpe3JldHVybiEhdGhpcy50aWxlcy5maW5kKHI9PnIudGFiSW5kZXg+LTEpfSxzdHlsZXMoKXtyZXR1cm57bWF4SGVpZ2h0OnRoaXMuY2FsY3VsYXRlZE1heEhlaWdodCxtaW5XaWR0aDp0aGlzLmNhbGN1bGF0ZWRNaW5XaWR0aCxtYXhXaWR0aDp0aGlzLmNhbGN1bGF0ZWRNYXhXaWR0aCx0b3A6dGhpcy5jYWxjdWxhdGVkVG9wLGxlZnQ6dGhpcy5jYWxjdWxhdGVkTGVmdCx0cmFuc2Zvcm1PcmlnaW46dGhpcy5vcmlnaW4sekluZGV4OnRoaXMuekluZGV4fHx0aGlzLmFjdGl2ZVpJbmRleH19fSx3YXRjaDp7aXNBY3RpdmUocil7cnx8KHRoaXMubGlzdEluZGV4PS0xKX0saXNDb250ZW50QWN0aXZlKHIpe3RoaXMuaGFzSnVzdEZvY3VzZWQ9cn0sbGlzdEluZGV4KHIsZSl7aWYociBpbiB0aGlzLnRpbGVzKXtjb25zdCB0PXRoaXMudGlsZXNbcl07dC5jbGFzc0xpc3QuYWRkKCJ2LWxpc3QtaXRlbS0taGlnaGxpZ2h0ZWQiKTtjb25zdCBzPXRoaXMuJHJlZnMuY29udGVudC5zY3JvbGxUb3Asbj10aGlzLiRyZWZzLmNvbnRlbnQuY2xpZW50SGVpZ2h0O3M+dC5vZmZzZXRUb3AtOD9BYSh0Lm9mZnNldFRvcC10LmNsaWVudEhlaWdodCx7YXBwT2Zmc2V0OiExLGR1cmF0aW9uOjMwMCxjb250YWluZXI6dGhpcy4kcmVmcy5jb250ZW50fSk6cytuPHQub2Zmc2V0VG9wK3QuY2xpZW50SGVpZ2h0KzgmJkFhKHQub2Zmc2V0VG9wLW4rdC5jbGllbnRIZWlnaHQqMix7YXBwT2Zmc2V0OiExLGR1cmF0aW9uOjMwMCxjb250YWluZXI6dGhpcy4kcmVmcy5jb250ZW50fSl9ZSBpbiB0aGlzLnRpbGVzJiZ0aGlzLnRpbGVzW2VdLmNsYXNzTGlzdC5yZW1vdmUoInYtbGlzdC1pdGVtLS1oaWdobGlnaHRlZCIpfX0sY3JlYXRlZCgpe3RoaXMuJGF0dHJzLmhhc093blByb3BlcnR5KCJmdWxsLXdpZHRoIikmJlRsKCJmdWxsLXdpZHRoIix0aGlzKX0sbW91bnRlZCgpe3RoaXMuaXNBY3RpdmUmJnRoaXMuY2FsbEFjdGl2YXRlKCl9LG1ldGhvZHM6e2FjdGl2YXRlKCl7dGhpcy51cGRhdGVEaW1lbnNpb25zKCkscmVxdWVzdEFuaW1hdGlvbkZyYW1lKCgpPT57dGhpcy5zdGFydFRyYW5zaXRpb24oKS50aGVuKCgpPT57dGhpcy4kcmVmcy5jb250ZW50JiYodGhyZXNzIiwKICAgICAgICAiaWQiOiAibTczcHJvZ3Jlc3MiLAogICAgICAgICJpbWFnZSI6ICJodHRwczovL3BsdWdpbnMub2N0b3ByaW50Lm9yZy9hc3NldHMvaW1nL3BsdWdpbnMvbTczcHJvZ3Jlc3MvbTczcHJvZ3Jlc3MuanBnIiwKICAgICAgICAiaXNfY29tcGF0aWJsZSI6IHsKICAgICAgICAgICJvY3RvcHJpbnQiOiB0cnVlLAogICAgICAgICAgIm9zIjogdHJ1ZSwKICAgICAgICAgICJweXRob24iOiB0cnVlCiAgICAgICAgfSwKICAgICAgICAibGljZW5zZSI6ICJBR1BMdjMiLAogICAgICAgICJwYWdlIjogImh0dHBzOi8vcGx1Z2lucy5vY3RvcHJpbnQub3JnL3BsdWdpbnMvbTczcHJvZ3Jlc3MvIiwKICAgICAgICAicHJpdmFjeXBvbGljeSI6IGZhbHNlLAogICAgICAgICJwdWJsaXNoZWQiOiAiMjAxOC0wMS0xNiAwMDowMDowMCArMDAwMCIsCiAgICAgICAgInN0YXRzIjogewogICAgICAgICAgImluc3RhbGxfZXZlbnRzX21vbnRoIjogMTcyLAogICAgICAgICAgImluc3RhbGxfZXZlbnRzX3dlZWsiOiAzNSwKICAgICAgICAgICJpbnN0YW5jZXNfbW9udGgiOiAyNzkxLAogICAgICAgICAgImluc3RhbmNlc193ZWVrIjogMjAzNAogICAgICAgIH0sCiAgICAgICAgInRpdGxlIjogIk03MyBQcm9ncmVzcyIKICAgICAgfSwKICAgICAgewogICAgICAgICJhYmFuZG9uZWQiOiBmYWxzZSwKICAgICAgICAiYXJjaGl2ZSI6ICJodHRwczovL2dpdGh1Yi5jb20vZ3J1dmluL09jdG9QcmludC1XZWJjYW1UYWIvYXJjaGl2ZS9tYXN0ZXIuemlwIiwKICAgICAgICAiYXV0aG9yIjogIlN2ZW4gTG9ocm1hbm4gKG9yaWdpbmFsKSwgQnJ5YW4gSi4gUmVudG91bCIsCiAgICAgICAgImNvbXBhdGliaWxpdHkiOiB7CiAgICAgICAgICAib2N0b3ByaW50IjogWwogICAgICAgICAgICAiMS4zLjAiCiAgICAgICAgICBdLAogICAgICAgICAgIm9zIjogW10sCiAgICAgICAgICAicHl0aG9uIjogIj49Mi43LDw0IgogICAgICAgIH0sCiAgICAgICAgImRlc2NyaXB0aW9uIjogIk1vdmVzIHRoZSB3ZWJjYW0gc3RyZWFtIGZyb20gQ29udHJvbCB0YWIgdG8gaXRzIG93biBXZWJjYW0gdGFiLiIsCiAgICAgICAgImZvbGxvd19kZXBlbmRlbmN5X2xpbmtzIjogZmFsc2UsCiAgICAgICAgImdpdGh1YiI6IHsKICAgICAgICAgICJpc3N1ZXMiOiB7CiAgICAgICAgICAgICJjbG9zZWQiOiAxMiwKICAgICAgICAgICAgIm9wZW4iOiAxCiAgICAgICAgICB9LAogICAgICAgICAgImxhc3RfcHVzaCI6ICIyMDIzLTA2LTIwIDAwOjUyOjU2ICswMDAwIiwKICAgICAgICAgICJsYXRlc3RfcmVsZWFzZSI6IHsKICAgICAgICAgICAgImRhdGUiOiAiMjAyMy0wNi0yMCAwMToxMDozNCArMDAwMCIsCiAgICAgICAgICAgICJuYW1lIjogIlN1cHBvcnQgZm9yIE9jdG9QcmludCAxLjkuMCBSRUxFQVNFIGJ5IGFsZXgxcGxhdG9uIiwKICAgICAgICAgICAgInRhZyI6ICIwLjMuMSIKICAgICAgICAgIH0sCiAgICAgICAgICAicmVsZWFzZXMiOiA4LAogICAgICAgICAgInN0YXJzIjogNgogICAgICAgIH0sCiAgICAgICAgImhvbWVwYWdlIjIuOTI3IFk0NS44MTggRS4wMDgyOQpHMSBYNjMuMjc1IFk0NS43MDQgRS4wMDk1NApHMSBYNjMuNTI3IFk0NS40NzUgRS4wMDg4NwpHMSBYNjMuNjYzIFk0NS40NDMgRS4wMDM2NApHMSBYNjQuNTYyIFk0NC45MzcgRS4wMjY4OApHMSBYNjQuNjkxIFk0NC45MTcgRS4wMDM0CkcxIFg2NS4wNjkgWTQ0LjUzNyBFLjAxMzk3CkcxIFg2NS40MzkgWTQ0LjIyNyBFLjAxMjU4CkcxIFg2NS42MjcgWTQ0LjAwMSBFLjAwNzY2CkcxIFg2Ni4wMzIgWTQzLjY2MiBFLjAxMzc2CkcxIFg2Ni4zMDMgWTQzLjE4MyBFLjAxNDM0CkcxIFg2Ni40NjkgWTQyLjk5NyBFLjAwNjUKRzEgWDY2LjY3NCBZNDIuNjUgRS4wMTA1CkcxIFg2Ny40NDcgWTQxLjg3OSBFLjAyODQ1CkcxIFg2Ny44MzQgWTQxLjI3NiBFLjAxODY3CkcxIFg2OC4yMTQgWTQwLjg3OSBFLjAxNDMyCkcxIFg2OC4yNzcgWTQwLjcyOCBFLjAwNDI2CkcxIFg2OC40OTQgWTQwLjQ0OSBFLjAwOTIxCkcxIFg2OS4xNzEgWTM5Ljk1MiBFLjAyMTg4CkcxIFg2OS41NDIgWTM5LjU1MSBFLjAxNDIzCkcxIFg3MC4wNTggWTM5LjIxNiBFLjAxNjAzCkcxIFg3MC4zMDEgWTM4Ljk2MSBFLjAwOTE4CkcxIFg3MC43OTcgWTM4LjcyNyBFLjAxNDI5CkcxIFg3MS4wNzMgWTM4LjUxOSBFLjAwOQpHMSBYNzEuMjk5IFkzOC40NTggRS4wMDYxCkcxIFg3MS43NTggWTM4LjExIEUuMDE1MDEKRzEgWDcyLjM3NSBZMzcuODgzIEUuMDE3MTMKRzEgWDcyLjYyNiBZMzcuNjk3IEUuMDA4MTQKRzEgWDczLjM2MSBZMzcuNDA2IEUuMDIwNgpHMSBYNzMuNTc1IFkzNy4xOTMgRS4wMDc4NwpHMSBYNzMuOTE2IFkzNy4wNTMgRS4wMDk2CkcxIFg3NC4xOTcgWTM2Ljc5IEUuMDEwMDMKRzEgWDc0LjQ0OSBZMzYuNjY2IEUuMDA3MzIKRzEgWDc0Ljc0MSBZMzYuNDc1IEUuMDA5MDkKRzEgWDc0Ljg3NSBZMzYuMjY2IEUuMDA2NDcKRzEgWDc1LjA1OCBZMzYuMTY1IEUuMDA1NDUKRzEgWDc1LjIzNyBZMzUuOTcyIEUuMDA2ODYKRzEgWDc1Ljc2MSBZMzUuNjMzIEUuMDE2MjYKRzEgWDc2LjIyMSBZMzUuMDExIEUuMDIwMTYKRzEgWDc2LjQxNiBZMzQuODY3IEUuMDA2MzIKRzEgWDc2LjU0OSBZMzQuNTYzIEUuMDA4NjUKRzEgWDc2LjcgWTM0LjQwMiBFLjAwNTc1CkcxIFg3Ni43OSBZMzQuMDM1IEUuMDA5ODUKRzEgWDc2Ljk4NCBZMzMuODQyIEUuMDA3MTMKRzEgWDc3LjMxOCBZMzMuMTg3IEUuMDE5MTYKRzEgWDc3LjMwNCBZMzIuOTE1IEUuMDA3MQpHMSBYNzcuNjQ3IFkzMi4yODkgRS4wMTg2CkcxIFg3Ny45MTggWTMxLjk5NCBFLjAxMDQ0CkcxIFg3OC4wNzUgWTMxLjYzNCBFLjAxMDIzCkcxIFg3OC42MzggWTMxLjA3NiBFLjAyMDY1CkcxIFg3OS4wMTkgWTMwLjUyNiBFLjAxNzQzCkcxIFg3OS4zMTYgWTMwLjE5MSBFLjAxMTY3CkcxIFg3OS41NTUgWTMwLjE4OCBFLjAwNjIzCkcxIFg3OS44MzEgWTMwLjA0OSBFLjAwODA1CkcxIFg3OS45NjEgWTI5Ljg3MyBFLjAwNTcKRzEgWDgwLjI3IFkyOS43MDEgRS4wMDkyMQpHMSBYODAuOTA2IFkyOS41MzQgRS4wMTcxMwpHMSBYODEuMzUzIFkyOS4zb24gSShpKXtmb3IodmFyIHQ9MTt0PGFyZ3VtZW50cy5sZW5ndGg7dCsrKXt2YXIgcj1udWxsIT1hcmd1bWVudHNbdF0/YXJndW1lbnRzW3RdOnt9LGU9T2JqZWN0LmtleXMocik7ImZ1bmN0aW9uIj09dHlwZW9mIE9iamVjdC5nZXRPd25Qcm9wZXJ0eVN5bWJvbHMmJihlPWUuY29uY2F0KE9iamVjdC5nZXRPd25Qcm9wZXJ0eVN5bWJvbHMocikuZmlsdGVyKGZ1bmN0aW9uKHQpe3JldHVybiBPYmplY3QuZ2V0T3duUHJvcGVydHlEZXNjcmlwdG9yKHIsdCkuZW51bWVyYWJsZX0pKSksZS5mb3JFYWNoKGZ1bmN0aW9uKHQpe3ZhciBlLG4sbztlPWksbz1yW249dF0sbiBpbiBlP09iamVjdC5kZWZpbmVQcm9wZXJ0eShlLG4se3ZhbHVlOm8sZW51bWVyYWJsZTohMCxjb25maWd1cmFibGU6ITAsd3JpdGFibGU6ITB9KTplW25dPW99KX1yZXR1cm4gaX1mdW5jdGlvbiBsKHQsZSl7aWYobnVsbD09dClyZXR1cm57fTt2YXIgbixvLGk9ZnVuY3Rpb24odCxlKXtpZihudWxsPT10KXJldHVybnt9O3ZhciBuLG8saT17fSxyPU9iamVjdC5rZXlzKHQpO2ZvcihvPTA7bzxyLmxlbmd0aDtvKyspbj1yW29dLDA8PWUuaW5kZXhPZihuKXx8KGlbbl09dFtuXSk7cmV0dXJuIGl9KHQsZSk7aWYoT2JqZWN0LmdldE93blByb3BlcnR5U3ltYm9scyl7dmFyIHI9T2JqZWN0LmdldE93blByb3BlcnR5U3ltYm9scyh0KTtmb3Iobz0wO288ci5sZW5ndGg7bysrKW49cltvXSwwPD1lLmluZGV4T2Yobil8fE9iamVjdC5wcm90b3R5cGUucHJvcGVydHlJc0VudW1lcmFibGUuY2FsbCh0LG4pJiYoaVtuXT10W25dKX1yZXR1cm4gaX1mdW5jdGlvbiBlKHQpe3JldHVybiBmdW5jdGlvbih0KXtpZihBcnJheS5pc0FycmF5KHQpKXtmb3IodmFyIGU9MCxuPW5ldyBBcnJheSh0Lmxlbmd0aCk7ZTx0Lmxlbmd0aDtlKyspbltlXT10W2VdO3JldHVybiBufX0odCl8fGZ1bmN0aW9uKHQpe2lmKFN5bWJvbC5pdGVyYXRvciBpbiBPYmplY3QodCl8fCJbb2JqZWN0IEFyZ3VtZW50c10iPT09T2JqZWN0LnByb3RvdHlwZS50b1N0cmluZy5jYWxsKHQpKXJldHVybiBBcnJheS5mcm9tKHQpfSh0KXx8ZnVuY3Rpb24oKXt0aHJvdyBuZXcgVHlwZUVycm9yKCJJbnZhbGlkIGF0dGVtcHQgdG8gc3ByZWFkIG5vbi1pdGVyYWJsZSBpbnN0YW5jZSIpfSgpfWZ1bmN0aW9uIHQodCl7aWYoInVuZGVmaW5lZCIhPXR5cGVvZiB3aW5kb3cmJndpbmRvdy5uYXZpZ2F0b3IpcmV0dXJuISFuYXZpZ2F0b3IudXNlckFnZW50Lm1hdGNoKHQpfXZhciB3PXQoLyg/OlRyaWRlbnQuKnJ2WyA6XT8xMVwufG1zaWV8aWVtb2JpbGV8V2luZG93cyBQaG9uZSkvaSksRT10KC9FZGdlL2kpLGM9dCgvZmlyZWZveC9pKSx1PXQoL3NhZmFyaS9pKSYmIXQoL2Nocm9tZS9pKSYmIXQoL2FuZHJvaWQvaSksbj10KC9pUChhZHxvZHxob25lKS9pKSxpPXQoL2Nocm9tZS9pKSYmdCgvYW5kcm9pZC9pKSxyPXtjYXB0dXJlOiExLHBhc3NpdmU6ITF9O2Z1bmN0aW9uIGQodCxlLG4pe3QuYWRkRXZlbnRMaXN0ZW5lcihlLG4sIXcmJnIpfWZ1bmN0aW9uIHModCxlLG4pe3QucmVtb3ZlRXZlbnRhU2xpY2VyIiwgInNsaWNlcl92ZXJzaW9uIjogIjIuNi4xK3dpbjY0IiwgImdjb2RlX3N0YXJ0X2J5dGUiOiAxMjU4OCwgImdjb2RlX2VuZF9ieXRlIjogODMxNDgwLCAib2JqZWN0X2hlaWdodCI6IDI1LjEsICJlc3RpbWF0ZWRfdGltZSI6IDEyMzQsICJub3p6bGVfZGlhbWV0ZXIiOiAwLjQsICJsYXllcl9oZWlnaHQiOiAwLjMsICJmaXJzdF9sYXllcl9oZWlnaHQiOiAwLjIsICJmaXJzdF9sYXllcl9leHRyX3RlbXAiOiAyMTUuMCwgImZpcnN0X2xheWVyX2JlZF90ZW1wIjogNjAuMCwgImZpbGFtZW50X25hbWUiOiAiUHJ1c2FtZW50IFBMQSIsICJmaWxhbWVudF90eXBlIjogIlBMQSIsICJmaWxhbWVudF90b3RhbCI6IDI0NTIuOTYsICJmaWxhbWVudF93ZWlnaHRfdG90YWwiOiA3LjMyLCAidGh1bWJuYWlscyI6IFt7IndpZHRoIjogMzIsICJoZWlnaHQiOiAyNCwgInNpemUiOiAxNTgzLCAicmVsYXRpdmVfcGF0aCI6ICIudGh1bWJzL1NoYXBlLUN5bGluZGVyXzAuNG5fMC4zbW1fUExBX01LM1NfMjFtLTMyeDMyLnBuZyJ9LCB7IndpZHRoIjogMTYwLCAiaGVpZ2h0IjogMTIwLCAic2l6ZSI6IDg4MjgsICJyZWxhdGl2ZV9wYXRoIjogIi50aHVtYnMvU2hhcGUtQ3lsaW5kZXJfMC40bl8wLjNtbV9QTEFfTUszU18yMW0tMTYweDEyMC5wbmcifV19LCAicHJpbnRfZHVyYXRpb24iOiAwLjAsICJzdGF0dXMiOiAiY2FuY2VsbGVkIiwgInN0YXJ0X3RpbWUiOiAxNzA5ODQ5MTU4LjgzMzE5MzMsICJ0b3RhbF9kdXJhdGlvbiI6IDIxMS42MTkyNDYwNTkwMDI3NiwgImpvYl9pZCI6ICIwMDAwMDAiLCAiZXhpc3RzIjogZmFsc2V9XX0sICJpZCI6IDM3ODU4fS8vIEpTIGFzc2V0cyBmb3IgcGx1Z2luIFByaW50VGltZUdlbml1cwooZnVuY3Rpb24gKCkgewogICAgdHJ5IHsKICAgICAgICAvLyBzb3VyY2U6IHBsdWdpbi9QcmludFRpbWVHZW5pdXMvanMvUHJpbnRUaW1lR2VuaXVzLmpzCiAgICAgICAgLyoKICAgICAgICAgKiBWaWV3IG1vZGVsIGZvciBPY3RvUHJpbnQtUHJpbnRUaW1lR2VuaXVzCiAgICAgICAgICoKICAgICAgICAgKiBBdXRob3I6IEV5YWwKICAgICAgICAgKiBMaWNlbnNlOiBBR1BMdjMKICAgICAgICAgKi8KICAgICAgICAkKGZ1bmN0aW9uKCkgewogICAgICAgICAgZnVuY3Rpb24gUHJpbnRUaW1lR2VuaXVzVmlld01vZGVsKHBhcmFtZXRlcnMpIHsKICAgICAgICAgICAgdmFyIHNlbGYgPSB0aGlzOwogICAgICAgIAogICAgICAgICAgICBzZWxmLnNldHRpbmdzVmlld01vZGVsID0gcGFyYW1ldGVyc1swXTsKICAgICAgICAgICAgc2VsZi5wcmludGVyU3RhdGVWaWV3TW9kZWwgPSBwYXJhbWV0ZXJzWzFdOwogICAgICAgICAgICBzZWxmLmZpbGVzVmlld01vZGVsID0gcGFyYW1ldGVyc1syXTsKICAgICAgICAgICAgc2VsZi5zZWxlY3RlZEdjb2RlcyA9IGtvLm9ic2VydmFibGUoKTsKICAgICAgICAgICAgc2VsZi5wcmludF9oaXN0b3J5ID0ga28ub2JzZXJ2YWJsZUFycmF5KCk7CiAgICAgICAgICAgIHNlbGYuc2V0dGluZ3NfdmlzaWJsZSA9IGtvLm9ic2VydmFibGUoZmFsc2UpOwogIC4wMTY1MQpHMSBYMTEyLjUwNyBZMTEwLjA1MyBFLjAwOTg3CkcxIFgxMTIuMTEgWTEwOC45MzUgRS4wMTY2OApHMSBYMTExLjkyMyBZMTA4LjI2IEUuMDA5ODUKRzEgWDExMS43NzEgWTEwNy41NzIgRS4wMDk5MQpHMSBYMTExLjY1NCBZMTA2Ljg3NiBFLjAwOTkzCkcxIFgxMTEuNTc0IFkxMDYuMTc1IEUuMDA5OTIKRzEgWDExMS41MzEgWTEwNS40NyBFLjAwOTkzCkcxIFgxMTEuNTI1IFkxMDQuNzY1IEUuMDA5OTIKRzEgWDExMS41NTYgWTEwNC4wNTkgRS4wMDk5NApHMSBYMTExLjYxMyBZMTAzLjQ0NiBFLjAwODY2CkcxIFgxMTEuNjg5IFkxMDIuODkzIEUuMDA3ODUKRzEgWDExMS44MTggWTEwMi4xOTggRS4wMDk5NApHMSBYMTExLjk4MiBZMTAxLjUxMiBFLjAwOTkyCkcxIFgxMTIuMTgzIFkxMDAuODM1IEUuMDA5OTMKRzEgWDExMi40MTggWTEwMC4xNyBFLjAwOTkyCkcxIFgxMTIuNjUzIFk5OS42IEUuMDA4NjcKRzEgWDExMi44ODYgWTk5LjA5MyBFLjAwNzg1CkcxIFgxMTMuMjEzIFk5OC40NjYgRS4wMDk5NApHMSBYMTEzLjU3MSBZOTcuODU4IEUuMDA5OTIKRzEgWDExMy45NiBZOTcuMjcgRS4wMDk5MgpHMSBYMTE0LjM4IFk5Ni43MDIgRS4wMDk5MwpHMSBYMTE0Ljc3MSBZOTYuMjI2IEUuMDA4NjYKRzEgWDExNS4xNDMgWTk1LjgxIEUuMDA3ODUKRzEgWDExNS42MzkgWTk1LjMwNSBFLjAwOTk1CkcxIFgxMTYuNTE4IFk5NC41MjcgRS4wMTY1MQpHMSBYMTE3LjA3OCBZOTQuMDk3IEUuMDA5OTMKRzEgWDExNy42NiBZOTMuNjk3IEUuMDA5OTMKRzEgWDExOC4yNjIgWTkzLjMyOCBFLjAwOTkzCkcxIFgxMTguODAzIFk5My4wMzMgRS4wMDg2NwpHMSBYMTE5LjMwMyBZOTIuNzg2IEUuMDA3ODQKRzEgWDExOS45NTEgWTkyLjUwNCBFLjAwOTk0CkcxIFgxMjAuNjA3IFk5Mi4yNTkgRS4wMDk4NQpHMSBYMTIxLjc0NSBZOTEuOTIyIEUuMDE2NjkKRzEgWDEyMi40MjggWTkxLjc3MSBFLjAwOTg0CkcxIFgxMjMuMTI0IFk5MS42NTQgRS4wMDk5MwpHMSBYMTIzLjgyNSBZOTEuNTc0IEUuMDA5OTIKRzEgWDEyNC41MyBZOTEuNTMxIEUuMDA5OTMKRzEgWDEyNS4xNDcgWTkxLjUyNCBFLjAwODY4CkcxIFgxMjUuNzA0IFk5MS41NDEgRS4wMDc4NApHMSBYMTI2LjQwOSBZOTEuNTk3IEUuMDA5OTUKRzEgWDEyNy4xMDggWTkxLjY4OSBFLjAwOTkyCkcxIFgxMjcuODAyIFk5MS44MTggRS4wMDk5MwpHMSBYMTI4LjQ4OCBZOTEuOTgyIEUuMDA5OTIKRzEgWDEyOS4xMDIgWTkyLjE2NCBFLjAwOTAxCkcxIFgxMjkuMjkxIFk5MS43ODkgRjkwMDAKO1RZUEU6RXh0ZXJuYWwgcGVyaW1ldGVyCjtXSURUSDowLjQxOTk5OQpHMSBGMTUwMApHMSBYMTMwLjQyOSBZOTIuMjE1IEUuMDE2MjgKRzEgWDEzMS4wODkgWTkyLjUxNiBFLjAwOTcyCkcxIFgxMzEuNzM0IFk5Mi44NTIgRS4wMDk3NApHMSBYMTMyLjM2MSBZOTMuMjIxIEUuMDA5NzUKRzEgWDEzMi45NjUgWTkzLjYyMSBFLjAwOTcxCkcxIFgxMzMuOTMgWTk0LjM2MSBFLjAxNjI5CkcxIFgxMzQuNDczIFk5NC44NDIgRS4wMDk3MgpHMSBYMTM0Ljk5MiBZOTUuMzUxMzguODYgWTEwNC4xNjQgRS4wMDk4CkcxIFgxMzguODg4IFkxMDUuMjQyIEUuMDE0NDUKRzEgWDEzOC44NTYgWTEwNS45NjkgRS4wMDk3NQpHMSBYMTM4Ljc4NyBZMTA2LjY5MSBFLjAwOTcyCkcxIFgxMzguNTg2IFkxMDcuODg4IEUuMDE2MjYKRzEgWDEzOC4yODIgWTEwOS4wNjMgRS4wMTYyNgpHMSBYMTM4LjA1MiBZMTA5Ljc1MSBFLjAwOTcyCkcxIFgxMzcuNzg2IFkxMTAuNDI2IEUuMDA5NzIKRzEgWDEzNy4yNjMgWTExMS41MjIgRS4wMTYyNwpHMSBYMTM2LjkwNiBZMTEyLjE1NCBFLjAwOTczCkcxIFgxMzYuNTE1IFkxMTIuNzY3IEUuMDA5NzQKRzEgWDEzNi4wOTMgWTExMy4zNTkgRS4wMDk3NApHMSBYMTM1LjY0IFkxMTMuOTI4IEUuMDA5NzQKRzEgWDEzNS4xNTggWTExNC40NzMgRS4wMDk3NQpHMSBYMTM0LjY5MiBZMTE0Ljk0OCBFLjAwODkyCkcxIFgxMzQuMjY3IFkxMTQuODc2IEY5MDAwCkcxIEUtMy41IEYzNjAwCjtXSVBFX1NUQVJUCkcxIEY3MjAwCkcxIFgxMzMuNzQgWTExNS43OTYgRS0uNjA1NTgKRzEgWDEzMy4xNjQgWTExNi4yMzcgRS0uMzQ0NTgKRzEgWDEzMi41NjYgWTExNi42NDggRS0uMzQ0NjcKRzEgWDEzMi4xOTQgWTExNi44NjcgRS0uMjA1MTcKO1dJUEVfRU5ECkcxIFgxMzUuNjAzIFkxMTIuMDM5IEY5MDAwCkcxIEU1IEYyNDAwCjtUWVBFOkludGVybmFsIGluZmlsbAo7V0lEVEg6MC40NApHMSBGMTU4NgpHMSBYMTM1LjE2NSBZMTEyLjY2IEUuMDEwNjkKRzEgWDEzNC43NSBZMTEzLjE4MiBFLjAwOTM4CkcxIFgxMzQuMzA5IFkxMTMuNjgxIEUuMDA5MzcKRzEgWDEzMy45OTggWTExMy45OTggRS4wMDYyNQpHMSBYMTE2LjAwMiBZOTYuMDAyIEUuMzU3OTIKRzEgWDExNS42OTEgWTk2LjMxOSBFLjAwNjI1CkcxIFgxMTUuMjUgWTk2LjgxOCBFLjAwOTM3CkcxIFgxMTQuODMzIFk5Ny4zNDIgRS4wMDk0MgpHMSBYMTE0LjM5OCBZOTcuOTYyIEUuMDEwNjUKRzEgRS0zLjUgRjM2MDAKO1dJUEVfU1RBUlQKRzEgRjcyMDAKRzEgWDExNC44MzMgWTk3LjM0MiBFLS4zNTk3NgpHMSBYMTE1LjI1IFk5Ni44MTggRS0uMzE4MQpHMSBYMTE1LjY5MSBZOTYuMzE5IEUtLjMxNjMyCkcxIFgxMTYuMDAyIFk5Ni4wMDIgRS0uMjEwOTQKRzEgWDExNi40NDEgWTk2LjQ0MSBFLS4yOTQ4OAo7V0lQRV9FTkQKRzEgWDExMi45MTkgWTEwMSBGOTAwMApHMSBFNSBGMjQwMApHMSBGMTU4NgpHMSBYMTEyLjk0NyBZMTAwLjkyIEUuMDAxMTkKRzEgWDEyOS4wNzQgWTExNy4wNDcgRS4zMjA3NQpHMSBYMTI4LjgwNCBZMTE3LjE0MiBFLjAwNDAzCkcxIFgxMzcuMTQ2IFkxMDguOCBFLjE2NTkxCkcxIFgxMzcuMDUyIFkxMDkuMDc5IEUuMDA0MTQKRzEgWDEyMC45MjYgWTkyLjk1MyBFLjMyMDczCkcxIFgxMjEuMTk2IFk5Mi44NTggRS4wMDQwMwpHMSBYMTEyLjg1MyBZMTAxLjIwMSBFLjE2NTkzCkcxIFgxMTIuNzA2IFkxMDEuNzAzIEUuMDA3MzYKRzEgWDExMi41NTMgWTEwMi4zNDIgRS4wMDkyNApHMSBYMTEyLjM2NSBZMTAzLjQ2MSBFLjAxNTk2CkcxIFgxMTIuMzQ0IFkxMDMuNjgyIEUuMDAzMTIKRzEuNDczIEUuMDA4MDQKRzEgWDgyLjQ4MiBZOTEuMjE1IEUuMDA4MTgKRzEgWDgyLjM1OCBZOTEuMTA2IEUuMDA0MwpHMSBYODIuMzAyIFk5MC45OTcgRS4wMDMxOQpHMSBYODIuMTQyIFk5MC44NDQgRS4wMDU3NwpHMSBYODEuNjA3IFk5MC4wOTQgRS4wMjQKRzEgWDgxLjU2MiBZODkuNzIxIEUuMDA5NzkKRzEgWDgxLjQxNCBZODkuNjA5IEUuMDA0ODQKRzEgWDgxLjUwNyBZODkuMzU4IEUuMDA2OTcKRzEgWDgxLjUwMyBZODkuMjEyIEUuMDAzODEKO1dJRFRIOjAuNDU1Nzk4CkcxIFg4MS4yNTMgWTg4LjY2OCBFLjAxNTgyCkcxIFg4MS4xMTggWTg4LjQyNyBFLjAwNzMKO1dJRFRIOjAuNDUzMzI0CkcxIFg4MS4yNjYgWTg4LjExNiBFLjAwOTA1CkcxIFg4MS41MjcgWTg4LjMzNiBFLjAwODk2CkcxIFg4MS44NDkgWTg4LjU2NCBFLjAxMDM2CjtXSURUSDowLjQ1MjU3MgpHMSBYODIuMTk5IFk4OC45NDkgRS4wMTM2NAo7V0lEVEg6MC40NDk5OTkKRzEgWDgyLjU0OSBZODkuMzU5IEUuMDE0MDUKRzEgWDgyLjY4MSBZODkuNDY1IEUuMDA0NDEKRzEgWDgyLjkzNiBZODkuNzgyIEUuMDEwNgpHMSBYODMuMDc3IFk4OS44ODYgRS4wMDQ1NwpHMSBYODMuMzQzIFk5MC4yMzQgRS4wMTE0MQpHMSBYODMuNjg5IFk5MC42MTIgRS4wMTMzNQpHMSBYODMuODM5IFk5MC43MzkgRS4wMDUxMgpHMSBYODQuMTA0IFk5MS4wODIgRS4wMTEyOQpHMSBYODQuODkzIFk5MS45NDYgRS4wMzA0OQpHMSBYODUuMzc1IFk5Mi41NDQgRS4wMjAwMQpHMSBYODYuMTIgWTkzLjQxIEUuMDI5NzYKRzEgWDg2LjE5OSBZOTMuNTg0IEUuMDA0OTgKRzEgWDg2LjMzMiBZOTMuNzExIEUuMDA0NzkKRzEgWDg2LjY1MiBZOTQuMTU1IEUuMDE0MjYKRzEgWDg2LjgxIFk5NC4xNjkgRS4wMDQxMwo7V0lEVEg6MC40MTkwMQpHMSBYODYuOTY4IFk5NC4xODQgRS4wMDM4MwpHMSBYODYuOTQgWTk0LjI2NSBFLjAwMjA3CjtXSURUSDowLjQ0OTk5OQpHMSBYODYuOTEyIFk5NC4zNDYgRS4wMDIyMwpHMSBYODcuMDczIFk5NC40NzIgRS4wMDUzMwpHMSBYODcuNDA0IFk5NC45NyBFLjAxNTU4CkcxIFg4Ny41NDcgWTk1LjMwMSBFLjAwOTM5CkcxIFg4Ny43OTIgWTk1Ljc0OSBFLjAxMzMKRzEgWDg3Ljk5NyBZOTYuMzEgRS4wMTU1NgpHMSBYODguMjMgWTk2LjgwNyBFLjAxNDMKRzEgWDg4LjYwNCBZOTcuODYzIEUuMDI5MTkKRzEgWDg4LjcwOCBZOTguMzU5IEUuMDEzMgpHMSBYODguODk5IFk5OC44NjYgRS4wMTQxMgpHMSBYODkuMzI3IFkxMDAuMzY1IEUuMDQwNjIKRzEgWDg5LjQ3NCBZMTAxLjExNyBFLjAxOTk2CkcxIFg4OS42MDggWTEwMS40ODcgRS4wMTAyNQpHMSBYODkuNzE3IFkxMDIuMTI3IEUuMDE2OTIKRzEgWDg5Ljc3NiBZMTAyLjY4NCBFLjAxNDU5CkcxIFg4OS43ODIgWTEwMi45ODYgRS4wMDc4NwpHMSBYODkuNzAxIFkxMDMuMjgzIEUuMDA4MDIKRzEgWDg5LjYyIFkxMDMuNzUzIEUuMDEyNDMKRzEgWDg5LjM4IFkxMDMuNjgxIEUuMDA2NTMKRzEgWDg5LjA4MiBZMTAzLjggRS4wMDgzNgpHMSBYODguOTc2IFkxMDMuOTI5IEUuMDA0MzUKRy42MDUgWTEzMi4wOTQgRS4wMTAzCkcxIFgxMTYuNTExIFkxMzIuNDE4IEUuMDA4NzkKRzEgWDExNi4xMzEgWTEzMy4wMDkgRS4wMTgzMQpHMSBYMTE2LjA5NyBZMTMzLjI1OSBFLjAwNjU3CkcxIFgxMTYuMDIyIFkxMzMuMzIxIEUuMDAyNTQKRzEgWDExNS45MjQgWTEzMy4zMjkgRS4wMDI1NgpHMSBYMTE1Ljg3MSBZMTMzLjQ3NiBFLjAwNDA3CkcxIFgxMTUuNzMzIFkxMzMuNTMgRS4wMDM4NgpHMSBYMTE1Ljc2NiBZMTMzLjg1NyBFLjAwODU2CkcxIFgxMTUuNzQ3IFkxMzQuMDU3IEUuMDA1MjMKRzEgWDExNS43MDggWTEzNC4yMDQgRS4wMDM5NgpHMSBYMTE1LjQ1NiBZMTM0LjU0NyBFLjAxMTA5CkcxIFgxMTUuMzY4IFkxMzQuNzc5IEUuMDA2NDcKRzEgWDExNS4yMTMgWTEzNC44NDIgRS4wMDQzNgpHMSBYMTE0Ljg5NCBZMTM0LjcyMiBFLjAwODg4CkcxIFgxMTQuODY0IFkxMzQuNTk0IEUuMDAzNDMKRzEgWDExNC41ODcgWTEzNC44NjQgRS4wMTAwOApHMSBYMTE0LjQ5MSBZMTM1LjAxMiBFLjAwNDYKRzEgWDExNC40ODIgWTEzNS4xNjggRS4wMDQwNwpHMSBYMTE0LjQyMyBZMTM1LjIyOCBFLjAwMjE5CkcxIFgxMTQuNDc1IFkxMzUuMzMxIEUuMDAzMDEKRzEgWDExNC4zNzggWTEzNS41OTEgRS4wMDcyMwpHMSBYMTE0LjA2NSBZMTM1Ljc4NyBFLjAwOTYyCkcxIFgxMTMuOSBZMTM1Ljc0OSBFLjAwNDQxCkcxIFgxMTMuODQ4IFkxMzUuNjIyIEUuMDAzNTgKRzEgWDExMy4wNzcgWTEzNi4yNDcgRS4wMjU4NgpHMSBYMTEyLjk1OSBZMTM2LjQzOSBFLjAwNTg3CkcxIFgxMTIuODgzIFkxMzYuNSBFLjAwMjU0CkcxIFgxMTIuNzgzIFkxMzYuNjg3IEUuMDA1NTMKRzEgWDExMi43NDIgWTEzNi42OTggRS4wMDExMQpHMSBYMTEyLjU0NyBZMTM3LjEyNSBFLjAxMjIzCkcxIFgxMTIuMzQ3IFkxMzcuMTcyIEUuMDA1MzUKRzEgWDExMi4yMTcgWTEzNy4xNjUgRS4wMDMzOQo7V0lEVEg6MC40NDgxNTcKRzEgWDExMi4xNTQgWTEzNy4wOTUgRS4wMDI0NAo7V0lEVEg6MC40MTkyMjYKRzEgWDExMi4wOTEgWTEzNy4wMjUgRS4wMDIyNwo7V0lEVEg6MC4zOTAyOTQKRzEgWDExMi4wOTQgWTEzNi45MzUgRS4wMDIwMQo7V0lEVEg6MC40MTEwOTUKRzEgWDExMi4wOSBZMTM3LjA4NyBFLjAwMzU5CjtXSURUSDowLjQ0OTk5OQpHMSBYMTEyLjA4NyBZMTM3LjIzOCBFLjAwMzk0CkcxIFgxMTEuOTYyIFkxMzcuNDc4IEUuMDA3MDUKRzEgWDExMS44MSBZMTM4LjA0NCBFLjAxNTI3CkcxIFgxMTEuNzQ1IFkxMzguMTQ3IEUuMDAzMTcKRzEgWDExMS41ODYgWTEzOC4yMjUgRS4wMDQ2MQpHMSBYMTExLjU0MyBZMTM4LjM1NSBFLjAwMzU3CkcxIFgxMTEuNjIzIFkxMzguNTI3IEUuMDA0OTQKRzEgWDExMS42MDQgWTEzOC44MDUgRS4wMDcyNgpHMSBYMTExLjU0NCBZMTM4LjkwMyBFLjAwMjk5CkcxIFgxMTEuNjI1IFkxMzkuMTM0IEUuMDA2MzgKRzEgWDExMS41OTMgWTEzOS43NTggRS4wMTYyOApHMSBYMTExLjY3MSBZMTQwLjM3MiBFLjAxNjEzCkcxIFgxMTEuNjExIFkxNDEuMDE1IEUuMDE2ODMKRzEgWDExMS40NzIgWTE0MS43RS4wMDg5MQpHMSBYMTIzLjgzNiBZODAuMjE2IEUuMDEwNDIKRzEgWDEyMy43MzggWTgwLjE4NCBFLjAwMjY5CkcxIFgxMjIuOTc0IFk4MC4xMzEgRS4wMTk5NQpHMSBYMTIyLjMxOCBZODAuMDIxIEUuMDE3MzMKRzEgWDEyMi4yMjEgWTc5Ljk0IEUuMDAzMjkKRzEgWDEyMS43MTIgWTc5Ljc2MSBFLjAxNDA2CkcxIFgxMjEuNjA2IFk3OS42OSBFLjAwMzMyCkcxIFgxMjEuMjY0IFk3OS42MTggRS4wMDkxMQpHMSBYMTIxLjIzMyBZNzkuNTUxIEUuMDAxOTIKRzEgWDEyMS4wMzMgWTc5LjQxMSBFLjAwNjM2CkcxIFgxMjAuNDI0IFk3OS4xMTIgRS4wMTc2OApHMSBYMTIwLjEzNyBZNzguODA3IEUuMDEwOTEKRzEgWDExOS42NDkgWTc4LjY4MiBFLjAxMzEzCkcxIFgxMTkuMTc2IFk3OC4zOTYgRS4wMTQ0CkcxIFgxMTguOTY3IFk3OC4xNiBFLjAwODIxCkcxIFgxMTguNjY2IFk3Ny43MzcgRS4wMTM1MwpHMSBYMTE4LjQyMSBZNzcuNTY0IEUuMDA3ODEKRzEgWDExOC4xNTMgWTc3LjUyNCBFLjAwNzA2CkcxIFgxMTcuOTk2IFk3Ny4zNTMgRS4wMDYwNQpHMSBYMTE3LjYxNCBZNzcuMTM1IEUuMDExNDYKRzEgWDExNy4zNzEgWTc2LjczIEUuMDEyMzEKRzEgWDExNy4xODMgWTc2LjUwNSBFLjAwNzY0CkcxIFgxMTYuOTI2IFk3Ni40MDMgRS4wMDcyCkcxIFgxMTYuODczIFk3Ni4yMDEgRS4wMDU0NApHMSBYMTE2LjY3MSBZNzUuOTc0IEUuMDA3OTIKRzEgWDExNi4yNTIgWTc1LjcyNyBFLjAxMjY3CkcxIFgxMTYuMTUxIFk3NC45MTQgRS4wMjEzNQpHMSBYMTE2LjEyNSBZNzQuMTQ1IEUuMDIwMDUKRzEgWDExNi4xOTcgWTczLjUzOCBFLjAxNTkzCkcxIFgxMTYuNTU0IFk3My43MzUgRS4wMTA2MgpHMSBYMTE2LjY2IFk3My43MzcgRS4wMDI3NgpHMSBYMTE2Ljc4MyBZNzMuODQgRS4wMDQxOApHMSBYMTE3LjE5NiBZNzMuOTc3IEUuMDExMzQKRzEgWDExNy41MDQgWTczLjkzMyBFLjAwODExCkcxIFgxMTcuNjk0IFk3My44MSBFLjAwNTkKRzEgWDExNy45MTggWTczLjQ1NiBFLjAxMDkyCkcxIFgxMTguMDM3IFk3My4xNDcgRS4wMDg2MwpHMSBYMTE4LjExNyBZNzIuNDQ3IEUuMDE4MzYKRzEgWDExOC4yMTMgWTcyLjQzNCBFLjAwMjUyCkcxIFgxMTguNDIxIFk3Mi4zMzcgRS4wMDU5OApHMSBYMTE4LjU4OCBZNzIuMTQ5IEUuMDA2NTUKRzEgWDExOC44ODQgWTcyLjE3OCBFLjAwNzc1CkcxIFgxMTkuMTcyIFk3Mi4wOTIgRS4wMDc4MwpHMSBYMTE5LjM4OCBZNzEuODggRS4wMDc4OQpHMSBYMTE5LjQ4NyBZNzEuNTUyIEUuMDA4OTMKRzEgWDExOS41NTkgWTcxLjU0NyBFLjAwMTg4CkcxIFgxMTkuNzg5IFk3MS40MzUgRS4wMDY2NwpHMSBYMTE5Ljk3IFk3MS4yMTEgRS4wMDc1CkcxIFgxMjAuMDgyIFk3MC44MDMgRS4wMTEwMgpHMSBYMTIwLjEyMSBZNzAuMzI5IEUuMDEyMzkKRzEgWDEyMC4zNTQgWTcwLjI2MSBFLjAwNjMyCkcxIFgxMjAuNTE1IFk3MC4xMSBFLjAwNTc1CkcxIFgxMjAuNjMzIFk2OS44MzIgRS4wMDc4NwpHMSBYMTIwLjczNiBZNjkuMjc1IEUuMDE0NzYKRzEgWDEyMC44MyBZNjkuMTMgRS4wMDQ1CkcuMDAzODEKRzEgWDEwMC4wODIgWTQ1LjA4NCBFLjAwMzI2CkcxIFg5OS42MzcgWTQ1LjQzNyBFLjAxNDgKRzEgWDk4Ljc1NSBZNDYuMjQ0IEUuMDMxMTUKRzEgWDk3LjQwNyBZNDcuNTU2IEUuMDQ5MDEKRzEgWDk2Ljg4NSBZNDguMTQ5IEUuMDIwNTgKRzEgWDk1Ljc2MSBZNDkuMTk2IEUuMDQwMDIKRzEgWDk0LjkxIFk1MC4xNDkgRS4wMzMyOQpHMSBYOTQuMDY4IFk1MS4wMzYgRS4wMzE4NwpHMSBYOTMuODc0IFk1MS4yODUgRS4wMDgyMgpHMSBYOTMuODE0IFk1MS40MTUgRS4wMDM3MwpHMSBYOTMuNzAyIFk1MS41MTQgRS4wMDM4OQpHMSBYOTMuNjM2IFk1MS42NDUgRS4wMDM4MgpHMSBYOTMuNTI0IFk1MS43MTkgRS4wMDM1CkcxIFg5My40NTEgWTUxLjg2OCBFLjAwNDMyCkcxIFg5My4zNCBZNTEuOTUzIEUuMDAzNjQKRzEgWDkyLjY3IFk1Mi44MTcgRS4wMjg0OQpHMSBYOTIuNDMgWTUzLjI0NCBFLjAxMjc2CkcxIFg5Mi4zNjcgWTUzLjI4MiBFLjAwMTkyCkcxIFg5Mi4yMzYgWTUzLjQ5NCBFLjAwNjQ5CkcxIFg5Mi4xMTIgWTUzLjYxNiBFLjAwNDUzCkcxIFg5MS43ODkgWTU0LjA5NiBFLjAxNTA3CkcxIFg5MS42MzYgWTU0LjQzMyBFLjAwOTY0CkcxIFg5MS41NzQgWTU0LjQ3NyBFLjAwMTk4CkcxIFg5MS40ODMgWTU0LjY2NiBFLjAwNTQ3CkcxIFg5MS4zMTkgWTU0Ljg1MiBFLjAwNjQ2CkcxIFg5MS4xMTQgWTU1LjMzMSBFLjAxMzU4CkcxIFg5MC43NjcgWTU1Ljg1NiBFLjAxNjQKRzEgWDkwLjYzNCBZNTYuMjA1IEUuMDA5NzMKRzEgWDkwLjU3OSBZNTYuMjQ4IEUuMDAxODIKRzEgWDkwLjM3MSBZNTYuNjUgRS4wMTE3OQpHMSBYOTAuMjUxIFk1Ny4wMjcgRS4wMTAzMQpHMSBYOTAuMTUxIFk1Ny4yMTIgRS4wMDU0OApHMSBYOTAuMDI0IFk1Ny42IEUuMDEwNjQKRzEgWDg5LjYwOCBZNTguNjY3IEUuMDI5ODQKRzEgWDg5LjUyOSBZNTkuMDA0IEUuMDA5MDIKRzEgWDg5LjQxNiBZNTkuMjc5IEUuMDA3NzUKRzEgWDg5LjI3NyBZNTkuOTA5IEUuMDE2ODEKRzEgWDg4Ljk0NCBZNjEuMTY1IEUuMDMzODYKRzEgWDg4LjcyNSBZNjIuNDYgRS4wMzQyMgpHMSBYODguNjU0IFk2My4wMjQgRS4wMTQ4MQpHMSBYODguNTc1IFk2My45NjUgRS4wMjQ2CkcxIFg4OC41NDggWTY0LjYwNSBFLjAxNjY5CkcxIFg4OC41NDQgWTY2LjQ2MSBFLjA0ODM2CkcxIFg4OC42MzcgWTY2Ljk2IEUuMDEzMjMKRzEgWDg4LjY5NSBZNjguMTE1IEUuMDMwMTMKRzEgWDg4LjYwNyBZNjguNDQ1IEUuMDA4OQpHMSBYODguNDU1IFk2OC41MjQgRS4wMDQ0NgpHMSBYODguMjMzIFk2OC40OTcgRS4wMDU4MwpHMSBYODguMTQzIFk2OC4zNzEgRS4wMDQwMwpHMSBYODguMDA5IFk2Ny44MzQgRS4wMTQ0MgpHMSBYODcuODMgWTY3LjI2NyBFLjAxNTQ5CkcxIFg4Ny41MzYgWTY1Ljg5MiBFLjAzNjY0CkcxIFg4Ny40NDQgWTY1LjIzIEUuMDE3NDEKRzEgWDg3LjM0OSBZNjQuMjIxIEUuMDI2NDEKRzEgWDg3LjM0NSBZNjMuNDcyIEUuMDE5NTIKRzEgWDg3LjI5OSBZNjIuNzM2IEUuMDE5MjEKRzEgWDg3LjQxMiBZNjIuMTgzIEUuMDE0NzEKR0Y0ODAwCkcxIFg3MC43MyBZODYuMTgyIEUuMDAzMzkKRzEgWDcwLjMzOCBZODUuODQzIEUuMDEwOTkKRzEgWDY5Ljg3MiBZODUuMjY0IEUuMDE1NzYKO1dJRFRIOjAuNDExMDk1CkcxIFg2OS43NzEgWTg1LjExNiBFLjAwNDIzCjtXSURUSDowLjQ0OTk5OQpHMSBYNjkuNjcxIFk4NC45NjggRS4wMDQ2NQpHMSBYNjYuNjk1IFk4MS45OTMgRS4xMDk2NAo7V0lQRV9TVEFSVApHMSBGODY0MApHMSBYNjkuMTQ1IFk4NC40NDIgRS0uOAo7V0lQRV9FTkQKRzEgWjEuMiBGNzIwCkcxIFg2Ny4zODggWTc5LjE0NCBGMTA4MDAKRzEgWi44IEY3MjAKRzEgRS44IEYyMTAwCkcxIEY0ODAwCkcxIFg2Ny4yODkgWTc5LjgzNSBFLjAxODE5CkcxIFg2Ny4xNzUgWTgwLjM2NyBFLjAxNDE4CkcxIFg2Ny4xMyBZODAuODcyIEUuMDEzMjEKRzEgWDY3LjA0NyBZODEuMzcgRS4wMTMxNQpHMSBYNjYuOTI4IFk4MS42MzUgRS4wMDc1NwpHMSBYNjkuMzM1IFk4NC4wNDIgRS4wODg2OQpHMSBYNjkuMjYgWTgzLjYyMiBFLjAxMTEyCkcxIFg2OS4zMzggWTgzLjEwOCBFLjAxMzU1CkcxIFg2OS41MzQgWTgyLjc2MSBFLjAxMDM4CkcxIFg2OS43MzQgWTgyLjYyNSBFLjAwNjMKRzEgWDY5LjU2IFk4Mi4wMDggRS4wMTY3CkcxIFg2OS4zNjMgWTgxLjE0NSBFLjAyMzA2CkcxIFg2OS4xNTcgWTc4Ljg2OCBFLjA1OTU3CkcxIFg2OS4wODQgWTc3LjI0NSBFLjA0MjMzCkcxIFg2OS4wOTQgWTc1LjM1NiBFLjA0OTIyCkcxIFg2OS4xNTMgWTc0LjgzNSBFLjAxMzY2CkcxIFg2OS4xODcgWTc0LjExMSBFLjAxODg5CkcxIFg2OS4yNzMgWTczLjYwNCBFLjAxMzQKRzEgWDY5LjQwMSBZNzIuNDU1IEUuMDMwMTIKRzEgWDY5LjU4MSBZNzEuNDk2IEUuMDI1NDIKRzEgWDY4Ljk4MyBZNzIuMjU1IEUuMDI1MTgKRzEgWDY4LjY5OSBZNzIuNjk5IEUuMDEzNzMKRzEgWDY4LjMyMSBZNzMuMDU1IEUuMDEzNTMKRzEgWDY3Ljk1OCBZNzMuNzU2IEUuMDIwNTcKRzEgWDY3Ljc0MiBZNzQuMjIxIEUuMDEzMzYKRzEgWDY3LjQ1MSBZNzQuNjQxIEUuMDEzMzEKRzEgWDY3LjMzIFk3NC45MzUgRS4wMDgyOApHMSBYNjcuNzU1IFk3NS4yNzIgRS4wMTQxMwpHMSBYNjguMDAzIFk3NS42NzggRS4wMTI0CkcxIFg2OC4xMTkgWTc2LjE4NSBFLjAxMzU1CkcxIFg2OC4xMDEgWTc2LjcwOSBFLjAxMzY2CkcxIFg2Ny42MDMgWTc4LjI5OCBFLjA0MzM5CkcxIFg2Ny40NCBZNzguOTQyIEUuMDE3MzEKRzEgWDY3Ljg0MSBZNzkuMDYgRS4wMTA4OQpHMSBYNjguMDEyIFk3OC4zODcgRS4wMTgwOQpHMSBYNjguMzg2IFk3Ny4yMjggRS4wMzE3Mwo7V0lEVEg6MC40ODk5NzkKRzEgWDY4LjQzNiBZNzcuMTI4IEUuMDAzMTkKO1dJRFRIOjAuNTI5OTU5CkcxIFg2OC40ODcgWTc3LjAyOCBFLjAwMzQ4CjtXSURUSDowLjU2OTkzOQpHMSBYNjguNTM3IFk3Ni45MjggRS4wMDM3NQo7V0lEVEg6MC42MDk5MTgKRzEgWDY4LjU4OCBZNzYuODI4IEUuMDA0MDQKRzEgWDY4LjYwNyBZNzYuOTM3IEUuMDAzOTkKO1dJRFRIOjAuNTY5OTM5CkcxIFg2OC42MjcgWTc3LjA0NiBFLjAwMzcyCjtXMmM3In0uZmEuZmEtdGhlcm1vbWV0ZXItMzpiZWZvcmV7Y29udGVudDoiXGYyYzgifS5mYS5mYS10aGVybW9tZXRlci0yOmJlZm9yZXtjb250ZW50OiJcZjJjOSJ9LmZhLmZhLXRoZXJtb21ldGVyLTE6YmVmb3Jle2NvbnRlbnQ6IlxmMmNhIn0uZmEuZmEtdGhlcm1vbWV0ZXItMDpiZWZvcmV7Y29udGVudDoiXGYyY2IifS5mYS5mYS1iYXRodHViOmJlZm9yZSwuZmEuZmEtczE1OmJlZm9yZXtjb250ZW50OiJcZjJjZCJ9LmZhLmZhLXdpbmRvdy1tYXhpbWl6ZSwuZmEuZmEtd2luZG93LXJlc3RvcmV7Zm9udC1mYW1pbHk6IkZvbnQgQXdlc29tZSA2IEZyZWUiO2ZvbnQtd2VpZ2h0OjQwMH0uZmEuZmEtdGltZXMtcmVjdGFuZ2xlOmJlZm9yZXtjb250ZW50OiJcZjQxMCJ9LmZhLmZhLXdpbmRvdy1jbG9zZS1ve2ZvbnQtZmFtaWx5OiJGb250IEF3ZXNvbWUgNiBGcmVlIjtmb250LXdlaWdodDo0MDB9LmZhLmZhLXdpbmRvdy1jbG9zZS1vOmJlZm9yZXtjb250ZW50OiJcZjQxMCJ9LmZhLmZhLXRpbWVzLXJlY3RhbmdsZS1ve2ZvbnQtZmFtaWx5OiJGb250IEF3ZXNvbWUgNiBGcmVlIjtmb250LXdlaWdodDo0MDB9LmZhLmZhLXRpbWVzLXJlY3RhbmdsZS1vOmJlZm9yZXtjb250ZW50OiJcZjQxMCJ9LmZhLmZhLWJhbmRjYW1wLC5mYS5mYS1lZXJjYXN0LC5mYS5mYS1ldHN5LC5mYS5mYS1ncmF2LC5mYS5mYS1pbWRiLC5mYS5mYS1yYXZlbHJ5e2ZvbnQtZmFtaWx5OiJGb250IEF3ZXNvbWUgNiBCcmFuZHMiO2ZvbnQtd2VpZ2h0OjQwMH0uZmEuZmEtZWVyY2FzdDpiZWZvcmV7Y29udGVudDoiXGYyZGEifS5mYS5mYS1zbm93Zmxha2Utb3tmb250LWZhbWlseToiRm9udCBBd2Vzb21lIDYgRnJlZSI7Zm9udC13ZWlnaHQ6NDAwfS5mYS5mYS1zbm93Zmxha2UtbzpiZWZvcmV7Y29udGVudDoiXGYyZGMifS5mYS5mYS1tZWV0dXAsLmZhLmZhLXN1cGVycG93ZXJzLC5mYS5mYS13cGV4cGxvcmVye2ZvbnQtZmFtaWx5OiJGb250IEF3ZXNvbWUgNiBCcmFuZHMiO2ZvbnQtd2VpZ2h0OjQwMH0KLmZhLWxheWVyc3twb3NpdGlvbjpyZWxhdGl2ZTtkaXNwbGF5OmlubGluZS1ibG9jazt3aWR0aDoxLjI1ZW07aGVpZ2h0OjEuMjVlbTtsaW5lLWhlaWdodDoxLjI1ZW07dGV4dC1hbGlnbjpjZW50ZXI7dmVydGljYWwtYWxpZ246bWlkZGxlfS5mYS1sYXllcnMuZmEtZnc+Knt3aWR0aDoxMDAlO2hlaWdodDoxMDAlfS5mYS1sYXllcnM+LmZhLC5mYS1sYXllcnM+LmZhLWxheWVycy10ZXh0LC5mYS1sYXllcnM+LmZhYiwuZmEtbGF5ZXJzPi5mYWQsLmZhLWxheWVycz4uZmFsLC5mYS1sYXllcnM+LmZhciwuZmEtbGF5ZXJzPi5mYXN7cG9zaXRpb246YWJzb2x1dGU7bGVmdDowO3dpZHRoOjEwMCU7bGluZS1oZWlnaHQ6aW5oZXJpdDt0ZXh0LWFsaWduOmNlbnRlcn0uZmEtbGF5ZXJzIFtkYXRhLWZhLXRyYW5zZm9ybSo9J3VwLTEnXXt0b3A6LTYuMjUlfS5mYS1sYXllcnMgW2RhdGEtZmEtdHJhbnNmb3JtKj0ndXAtMidde3RvcDotMTIuNSV9LmZhLWxheWVycyBbZGF0YS1mYS10cmFuc2Zvcm0qPSd1cC0zJ117dG9wOi0xOC43NSV9LmY3ODggWTE4OC44MDkgRS4wMDQwOQpHMSBYMTAxLjg0NCBZMTg4LjkxOSBFLjAwMzIyCkcxIFgxMDEuOTcyIFkxODguOTk1IEUuMDAzODgKRzEgWDEwMi4wNzUgWTE4OS4yNzEgRS4wMDc2OApHMSBYMTAyLjIxOCBZMTg5LjQ4MSBFLjAwNjYyCkcxIFgxMDIuMjI5IFkxODkuNjA2IEUuMDAzMjcKRzEgWDEwMi4zMTUgWTE4OS43MzYgRS4wMDQwNgpHMSBYMTAyLjYyOCBZMTg5Ljc4NiBFLjAwODI2CkcxIFgxMDIuOTE1IFkxODkuODg5IEUuMDA3OTQKRzEgWDEwMy4yMjQgWTE5MC40MjQgRS4wMTYxCkcxIFgxMDMuNjE5IFkxOTAuNDg5IEUuMDEwNDMKRzEgWDEwMy44NjMgWTE5MC43NDYgRS4wMDkyMwpHMSBYMTA0LjA0NSBZMTkwLjc5OCBFLjAwNDkzCkcxIFgxMDQuMDc0IFkxOTAuODY1IEUuMDAxOQpHMSBYMTA0LjMyNCBZMTkxLjE0MiBFLjAwOTcyCkcxIFgxMDQuNTQxIFkxOTEuNDU3IEUuMDA5OTcKRzEgWDEwNC44NjMgWTE5MS41NjkgRS4wMDg4OApHMSBYMTA1LjA3MiBZMTkxLjg0NyBFLjAwOTA2CkcxIFgxMDUuNTQ0IFkxOTEuOTI0IEUuMDEyNDYKRzEgWDEwNS45MTcgWTE5Mi4wNzcgRS4wMTA1CkcxIFgxMDYuMzkzIFkxOTIuMDgyIEUuMDEyNApHMSBYMTA2LjQ4NCBZMTkyLjkxOCBFLjAyMTkxCkcxIFgxMDYuNTc4IFkxOTMuMTQ2IEUuMDA2NDMKRzEgWDEwNi41OTYgWTE5My40NDYgRS4wMDc4MwpHMSBYMTA2LjczMiBZMTk0LjAxNCBFLjAxNTIyCkcxIFgxMDYuOTIxIFkxOTQuNDgyIEUuMDEzMTUKRzEgWDEwNi45OSBZMTk0LjYwNyBFLjAwMzcyCkcxIFgxMDcuMDIgWTE5NC44MjEgRS4wMDU2MwpHMSBYMTA3LjIwMSBZMTk1LjAxNSBFLjAwNjkxCkcxIFgxMDcuNDMyIFkxOTUuMzQxIEUuMDEwNDEKRzEgWDEwNy41MDMgWTE5NS41ODcgRS4wMDY2NwpHMSBYMTA3LjYxOCBZMTk1LjY4OSBFLjAwNDAxCkcxIFgxMDcuNjU5IFkxOTUuODM2IEUuMDAzOTgKRzEgWDEwNy43OCBZMTk1Ljk0NiBFLjAwNDI2CkcxIFgxMDcuODUzIFkxOTYuMDczIEUuMDAzODIKRzEgWDEwNy45NzYgWTE5Ni4xNjYgRS4wMDQwMgpHMSBYMTA4LjA2NiBZMTk2LjI5NCBFLjAwNDA4CkcxIFgxMDguMTg5IFkxOTYuMzcyIEUuMDAzNzkKRzEgWDEwOC4yODQgWTE5Ni40OTEgRS4wMDM5NwpHMSBYMTA4LjQ1OSBZMTk2LjU3NiBFLjAwNTA3CkcxIFgxMDguNTExIFkxOTYuNjc4IEUuMDAyOTgKRzEgWDEwOC43NiBZMTk2Ljc4MyBFLjAwNzA0CkcxIFgxMDguNzk3IFkxOTYuODM3IEUuMDAxNzEKRzEgWDEwOS4xMjggWTE5Ni45NTggRS4wMDkxOApHMSBYMTA5LjU3OCBZMTk3LjIzNCBFLjAxMzc1CkcxIFgxMTAuMDc2IFkxOTcuMzg0IEUuMDEzNTUKRzEgWDExMC40MzMgWTE5Ny40MiBFLjAwOTM1CkcxIFgxMTAuNzg1IFkxOTcuMzk2IEUuMDA5MTkKRzEgWDExMS4xNDggWTE5Ny40ODUgRS4wMDk3NApHMSBYMTExLjQwOCBZMTk3LjQ4NiBFLjAwNjc3CkcxIFgxMTEuNzEyIFkxOTcuMzkxIEUuMDA4MwpHMSBYMTExLjgwNSBZMTk3LjI4NSBFLjAwMzY3CkcxIFgxMTEuOTYyIFkxOTYuODEgRS4wMTMwMwpHMSBYMTEyLjA3OCBZMUcxIFgzNC45MDggWTkwLjA5OSBFLjA1ODMyCkcxIFgzNC44ODUgWTg5LjIzNSBFLjAyMjUyCkcxIFgzNC44MzIgWTg5LjAwMyBFLjAwNjIKRzEgWDM0Ljg1IFk4OC41NjkgRS4wMTEzMgpHMSBYMzQuODg0IFk4OC40IEUuMDA0NDkKRzEgWDM0LjgyOSBZODguMDYxIEUuMDA4OTUKRzEgWDM0LjU5IFk4Ny45MjQgRS4wMDcxOApHMSBYMzQuNTUxIFk4Ny43NjEgRS4wMDQzNwpHMSBYMzQuNjQ1IFk4Ny42MjEgRS4wMDQzOQpHMSBYMzQuODMxIFk4Ny41MzkgRS4wMDUzCkcxIFgzNC44NDIgWTg3LjI4NCBFLjAwNjY1CkcxIFgzNC42NzUgWTg3LjA0NCBFLjAwNzYyCkcxIFgzNC43NjMgWTg2LjU0OSBFLjAxMzEKRzEgWDM0Ljg4OSBZODYuMzA5IEUuMDA3MDYKRzEgWDM0LjkxIFk4NS41OTggRS4wMTg1MwpHMSBYMzQuODk3IFk4NS4wMDMgRS4wMTU1MQpHMSBYMzQuOTY1IFk4NC4zNzIgRS4wMTY1NApHMSBYMzUuMDQyIFk4NC4xOTYgRS4wMDUwMQpHMSBYMzQuOTY5IFk4NC4wMTIgRS4wMDUxNgpHMSBYMzQuOTc0IFk4My43NDYgRS4wMDY5MwpHMSBYMzUuMDgzIFk4My42MDMgRS4wMDQ2OApHMSBYMzQuOTYzIFk4My40MjEgRS4wMDU2OApHMSBYMzQuOTkyIFk4My4wOTIgRS4wMDg2MQpHMSBYMzUuMDkgWTgyLjkzMyBFLjAwNDg3CkcxIFgzNS4xMDEgWTgyLjc0MiBFLjAwNDk4CkcxIFgzNS4xOTUgWTgyLjYgRS4wMDQ0NApHMSBYMzUuMDk1IFk4Mi40MTIgRS4wMDU1NQpHMSBYMzUuMTE2IFk4MS41ODkgRS4wMjE0NQpHMSBYMzUuMiBZODEuMzggRS4wMDU4NwpHMSBYMzUuMTQgWTgxLjE2NCBFLjAwNTg0CkcxIFgzNS4yMDEgWTgxLjAyIEUuMDA0MDcKRzEgWDM1LjI5MSBZODAuOTU2IEUuMDAyODgKRzEgWDM1LjMxOCBZNzkuNzkgRS4wMzAzOQpHMSBYMzUuNDI0IFk3OS41OCBFLjAwNjEzCkcxIFgzNS40ODMgWTc5LjU0NiBFLjAwMTc3CkcxIFgzNS40ODMgWTc5LjI1NCBFLjAwNzYxCkcxIFgzNS4zNTcgWTc5LjEyOCBFLjAwNDY0CkcxIFgzNS4zMTggWTc5LjAxMyBFLjAwMzE2CkcxIFgzNS4yOTggWTc4LjQ2OSBFLjAxNDE4CkcxIFgzNS4xNzIgWTc4LjIwOSBFLjAwNzUzCkcxIFgzNS4xODMgWTc3LjkxMiBFLjAwNzc0CkcxIFgzNS4yNjIgWTc3LjgwOCBFLjAwMzQKRzEgWDM1LjMyOSBZNzcuNzgzIEUuMDAxODYKRzEgWDM1LjM0NyBZNzcuMjc2IEUuMDEzMjIKRzEgWDM1LjE3NSBZNzcuMDIzIEUuMDA3OTcKRzEgWDM1LjIyNSBZNzYuODU3IEUuMDA0NTIKRzEgWDM1LjM4MiBZNzYuNzgxIEUuMDA0NTQKRzEgWDM1LjUyNiBZNzYuNzQ4IEUuMDAzODUKRzEgWDM1LjY4NSBZNzYuNDM3IEUuMDA5MQpHMSBYMzUuNzExIFk3NiBFLjAxMTQxCkcxIFgzNS44MDEgWTc1LjQ4NCBFLjAxMzY1CkcxIFgzNS44OTggWTc1LjM0NiBFLjAwNDQKRzEgWDM1Ljk1OCBZNzQuNzg1IEUuMDE0NwpHMSBYMzYuMDA1IFk3NC42NjEgRS4wMDM0NgpHMSBYMzYuMjk2IFk3NC4zMDQgRS4wMTIKRzEgWDM2LjM3OSBZNzMuMzIyIEUuMDI1NjgKRzEgWDM2LjUwOSBZNzMuMTI5IEUuMDA2MDYKRzEgWDM2LjUyMSBZNzIuNjc4IEUuWDE2Mi41NzUgWTE2OS4yNzYgRS4wMDU1MQpHMSBYMTYyLjk3NyBZMTY5LjM1MiBFLjAxMDY2CkcxIFgxNjMuMTc5IFkxNjkuNDQ1IEUuMDA1NzkKRzEgWDE2My44NzQgWTE2OS41MzcgRS4wMTgyNwpHMSBYMTY0LjMxOSBZMTY5LjY0OCBFLjAxMTk1CkcxIFgxNjUuNjg5IFkxNjkuNjc5IEUuMDM1NzEKRzEgWDE2Ni41NDggWTE2OS42MDQgRS4wMjI0NwpHMSBYMTY2Ljg1NSBZMTY5LjQ0NyBFLjAwODk4CkcxIFgxNjcuMjMyIFkxNjkuNDAxIEUuMDA5OQpHMSBYMTY3LjM5NCBZMTY5LjI1MSBFLjAwNTc1CkcxIFgxNjcuNzM0IFkxNjkuMjA5IEUuMDA4OTMKRzEgWDE2OC4wMDIgWTE2OS4wMjQgRS4wMDg0OQpHMSBYMTY4LjA4MiBZMTY5LjAwOSBFLjAwMjEyCkcxIFgxNjguMTg0IFkxNjguODg1IEUuMDA0MTgKRzEgWDE2OC41NjEgWTE2OC43NSBFLjAxMDQzCkcxIFgxNjggWTE2OC42MjggRS4wMTQ5NgpHMSBYMTY3LjU0MiBZMTY4LjQ2NSBFLjAxMjY3CkcxIFgxNjcuNDI3IFkxNjguMzkgRS4wMDM1OApHMSBYMTY2Ljk3NyBZMTY4LjIxNyBFLjAxMjU2CkcxIFgxNjYuNjY4IFkxNjguMTYgRS4wMDgxOQpHMSBYMTY2LjIxMiBZMTY3Ljk4NSBFLjAxMjczCkcxIFgxNjUuODE4IFkxNjcuODkyIEUuMDEwNTUKRzEgWDE2NS40MyBZMTY3LjcxMyBFLjAxMTEzCkcxIFgxNjQuNjcxIFkxNjcuNDkzIEUuMDIwNTkKRzEgWDE2NC4xOTQgWTE2Ny4yODcgRS4wMTM1NApHMSBYMTYzLjUwMSBZMTY3LjAyNiBFLjAxOTI5CkcxIFgxNjEuNTQgWTE2Ni4xOTMgRS4wNTU1MQpHMSBYMTYwLjkyNSBZMTY2LjAwMSBFLjAxNjc5CkcxIFgxNjAuODEyIFkxNjUuOTExIEUuMDAzNzYKRzEgWDE2MC43OCBZMTY1LjYxNCBFLjAwNzc4Ck0yMDQgUzEwMDAKRzEgWDE2MC42ODMgWTE2NS40NzYgRjEwODAwCjtXSVBFX1NUQVJUCkcxIEY4NjQwCkcxIFgxNjAuODI1IFkxNjUuNDQ2IEUtLjA0MDE2CkcxIFgxNjEuMjM1IFkxNjUuMyBFLS4xMDA0OQpHMSBYMTYxLjM1MyBZMTY1LjI5NSBFLS4wMjcyNwpHMSBYMTYxLjQ1MyBZMTY1LjE5NCBFLS4wMzI4MgpHMSBYMTYxLjk2MyBZMTY1LjA2NyBFLS4xMjEzNgpHMSBYMTYyLjI2OCBZMTY0LjkwNiBFLS4wNzk2NApHMSBYMTYzLjQ5OSBZMTY0LjQ2MSBFLS4zMDIyNApHMSBYMTYzLjcxIFkxNjQuMzQyIEUtLjA1NTkzCkcxIFgxNjMuODgyIFkxNjQuMzIxIEUtLjA0MDA5CjtXSVBFX0VORApHMSBaMi4xIEY3MjAKRzEgWDE4Mi4yNTIgWTE1OC43OTUgRjEwODAwCkcxIFoxLjcgRjcyMApHMSBFLjggRjIxMDAKTTIwNCBTODAwCkcxIEYxNTAwCkcxIFgxODIuMTQ1IFkxNTguNzkzIEUuMDAyNzkKRzEgWDE4MS45NDMgWTE1OC42MTQgRS4wMDcwMwo7V0lEVEg6MC40NDk2MDIKRzEgWDE4MS45MDYgWTE1OC4xNzEgRS4wMTE1Nwo7V0lEVEg6MC40Mjc2NzIKRzEgWDE4Mi4wMDUgWTE1Ny40MiBFLjAxODY4CjtXSURUSDowLjM5NjAxOQpHMSBYMTgyLjAzOSBZMTU3LjMxNCBFLjAwMjUzCkcxIFgxODIuMjE5IFkxNTYuNzc3IEUuMDEyODUKO1dJRFRIOjAuNDAzNzg0CkcxIFgxODIuNDk2IFkxNTYgWTExOS43NTMgRS4wMzUxOQpHMSBGMTQ4Ny4zMzUKRzEgWDEzMS4zMTUgWTExOS45MTEgRS4wMDcyNwpHMSBGMTQ2NC4yMzgKRzEgWDEzMS4xOTggWTEyMC4yNjEgRS4wMDk2MgpHMSBGMTUwMApHMSBYMTI5LjY1NCBZMTIwLjI2OSBFLjA0MDIzCkcxIFgxMjkuNTg0IFkxMjAuMjM5IEUuMDAxOTgKRzEgWDEyOS41ODUgWTExOC40NjcgRS4wNDYxNwpHMSBYMTI5LjYyMSBZMTE4LjQyIEUuMDAxNTQKRzEgWDEzMC42MDcgWTExOC4yOTIgRS4wMjU5MQpHMSBYMTMyLjAxMiBZMTE4LjA1NSBFLjAzNzEzCkcxIFgxMzMuMTY1IFkxMTcuNzk5IEUuMDMwNzcKRzEgWDEzNC4yMTYgWTExNy41MTUgRS4wMjgzNwpHMSBYMTM1LjEzMSBZMTE3LjA5NiBFLjAyNjIyCkcxIFgxMzUuNzA4IFkxMTYuNzI0IEUuMDE3ODkKRzEgWDEzNi4wOTIgWTExNi4zMTcgRS4wMTQ1OApHMSBYMTM2LjIyNSBZMTE1LjkzOSBFLjAxMDQ0CkcxIFgxMzYuMjEzIFk5NC4xNzggRS41NjcKCk0yMDQgUzEwMDAKRzEgWDEzNi4zNzggWTk0LjAxOSBGMTA4MDAKO1dJUEVfU1RBUlQKRzEgRjg2NDAKRzEgWDEzNi4yMDcgWTkzLjkyNSBFLS4wNTg0MwpHMSBYMTM1Ljk0NSBZOTMuNDkgRS0uMTE3MjUKRzEgWDEzNS41MDQgWTkzLjEyNyBFLS4xMzE4OQpHMSBYMTM0Ljg4IFk5Mi43NzggRS0uMTY1MDkKRzEgWDEzNC4yMTYgWTkyLjQ4NiBFLS4xNjc0OQpHMSBYMTMzLjU0OCBZOTIuMzA1IEUtLjE1OTg1CjtXSVBFX0VORApHMSBaMTUuNDUgRjcyMApHMSBYMTM1LjU2NCBZOTIuMDY2IEYxMDgwMApHMSBaMTUuMDUgRjcyMApHMSBFLjggRjIxMDAKO1RZUEU6U29saWQgaW5maWxsCkcxIEY0ODAwCkcxIFgxMzYuNzUxIFk5Mi40ODQgRS4wMzI3OQpHMSBYMTM3LjM5MSBZOTIuNzM4IEUuMDE3OTQKRzEgWDEzNy40MjggWTkyLjg0OCBFLjAwMzAyCkcxIFgxMzcuNjY5IFk5My4xNTggRS4wMTAyMwpHMSBYMTM4LjEyOCBZOTMuNDI5IEUuMDEzODkKRzEgWDEzOC43MDQgWTkzLjkxOCBFLjAxOTY5CkcxIFgxMzkuMDIgWTk0LjM0MiBFLjAxMzc4CkcxIFgxMzkuMTc0IFk5NC44MDQgRS4wMTI2OQpHMSBYMTM5LjE5IFk5NS42NDMgRS4wMjE4NgpHMSBYMTM5LjE4NyBZMTE0Ljk2MyBFLjUwMzQKRzEgWDEzOS4xMSBZMTE1LjQ2MyBFLjAxMzE4CkcxIFgxMzguODAzIFkxMTUuOTUzIEUuMDE1MDcKRzEgWDEzOC41NDMgWTExNi4yMjggRS4wMDk4NgpHMSBYMTM3LjY2MSBZMTE2LjkwMSBFLjAyODkxCkcxIFgxMzcuNDk0IFkxMTcuMjIzIEUuMDA5NDUKRzEgWDEzNi44MjEgWTExNy40OTIgRS4wMTg4OApHMSBYMTM1LjM0MyBZMTE4LjAwOCBFLjA0MDc5CkcxIFgxMzUuMTk4IFkxMTcuOTk5IEUuMDAzNzkKRzEgWDEzNS4yOSBZMTE3Ljg5MiBFLjAwMzY4CkcxIFgxMzYuMTI3IFkxMTcuMzk5IEUuMDI1MzEKRzEgWDEzNi41MyBZMTE3LjAyMiBFLjAxNDM4CkcxIFgxMzYuOCBZMTE2LjY2IEUuMDExNzcKRzEgWDEzNi45NDggWTExNi4zMDkgRS4wMDk5MwpHMSBYMTM3LjAxOSBZMTE1LjkzNSBFLjAwOTkyCkcxIFgxMzcuMDE5IFk5Ni4xNzEgRS41MTQ5NwpHMUdIVDowLjE1CjtCRUZPUkVfTEFZRVJfQ0hBTkdFCkc5MiBFMC4wCjsxMS43NQoKCjtXSVBFX1NUQVJUCkcxIEY4NjQwCkcxIFgxMjMuMzg3IFk5MS4yMzIgRS0uMDY2MjUKRzEgWDEyMy4wMzQgWTkxLjI3OCBFLS4wODIyCkcxIFgxMjIuNjgyIFk5MS4zMjQgRS0uMDgxOTcKRzEgWDEyMi41OTEgWTkxLjM2MiBFLS4wMjI3NwpHMSBYMTIyLjUwMSBZOTEuNCBFLS4wMjI1NgpHMSBYMTIyLjQxMSBZOTEuNDM4IEUtLjAyMjU2CkcxIFgxMjIuMzIxIFk5MS40NzcgRS0uMDIyNjUKRzEgWDEyMS41NDYgWTkxLjU4NSBFLS4xODA2OApHMSBYMTIwLjc3IFk5MS42OTQgRS0uMTgwOTQKRzEgWDEyMC4yNjYgWTkxLjc2NSBFLS4xMTc0Mgo7V0lQRV9FTkQKRzEgWjEyIEY3MjAKO0FGVEVSX0xBWUVSX0NIQU5HRQo7MTEuNzUKOyBwcmludGluZyBvYmplY3QgM2RiZW5jaHkuc3RsIGlkOjAgY29weSAwCkcxIFg5Ny4zNjkgWTEwMy40NTkgRjEwODAwCkcxIFoxMS43NSBGNzIwCkcxIEUuOCBGMjEwMApNMjA0IFM4MDAKO1RZUEU6RXh0ZXJuYWwgcGVyaW1ldGVyCjtXSURUSDowLjM5MTA2OApHMSBGMTUwMApHMSBYOTcuNzAxIFkxMDMuNTAzIEUuMDA3NQo7V0lEVEg6MC4zOTc4NjgKRzEgWDk4LjE5MiBZMTAzLjcyNCBFLjAxMjI4CjtXSURUSDowLjM5ODAyMQpHMSBYOTguNTM5IFkxMDQuMDM3IEUuMDEwNjYKO1dJRFRIOjAuNDA5MDIKRzEgWDk4Ljc3NCBZMTA0LjQyNSBFLjAxMDY2CjtXSURUSDowLjQxMTIyCkcxIFg5OC44NjkgWTEwNC43NiBFLjAwODIzCkcxIFg5OC44OSBZMTA1LjA0MiBFLjAwNjY4CjtXSURUSDowLjQxMjY0OApHMSBYOTguODQ5IFkxMDUuMzQxIEUuMDA3MTYKRzEgWDk4LjcwNSBZMTA1LjcxOSBFLjAwOTYKO1dJRFRIOjAuNDAwNDU5CkcxIFg5OC40MDIgWTEwNi4xMTMgRS4wMTE0Mgo7V0lEVEg6MC4zOTczMjcKRzEgWDk3Ljk3NSBZMTA2LjM5NCBFLjAxMTY0CjtXSURUSDowLjQwMTg5CkcxIFg5Ny41MjIgWTEwNi41MjggRS4wMTA4OQpHMSBYOTcuMTgxIFkxMDYuNTUgRS4wMDc4OAo7V0lEVEg6MC4zOTM0NzIKRzEgWDk2LjY2MSBZMTA2LjQzMiBFLjAxMjAxCjtXSURUSDowLjQwMzk3MwoKRzEgWDk2LjM0NiBZMTA2LjI2NiBFLjAwODI2CkcxIFg5Ni4wNTcgWTEwNi4wMiBFLjAwODgKCjtXSURUSDowLjQwNDQ1NAoKRzEgWDk1Ljc4MSBZMTA1LjU4MiBFLjAxMjAyCgo7V0lEVEg6MC40MTM4NDYKRzEgWDk1LjY4MSBZMTA1LjIzOCBFLjAwODUzCkcxIFg5NS42NjcgWTEwNC44NTkgRS4wMDkwMwo7V0lEVEg6MC40MDU1NTYKRzEgWDk1LjgwOCBZMTA0LjM1NCBFLjAxMjIxCjtXSURUSDowLjQxMDE1CkcxIFg5NS45NzQgWTEwNC4wODYgRS4wMDc0MwpHMSBYOTYuMzM0IFkxMDMuNzQzIEUuMDExNzIKO1dJRFRIOjAuMzkxOTU5CkcxIFg5Ni43OSBZMTAzLjUyMSBFLjAxMTM4CjtXSURUSDowLjM5MTM2OQpHMSBYOTcuMjk3IFkxMDMuNDUgRS4wMTE0Nwo7V0lEVEg6MC4zOTEwNjgKRzEgWDk3LjMwOSBZMTAzLjQ1MSBFLjAwMDI3Ck0yMDQgUzEwMDAKRzEgWDk3LjU1MiBZMTAzLjg0MSBGMTA4YXR1cy1iYXItc3R5bGUiIGNvbnRlbnQ9ImJsYWNrIiAvPgogICAgICAgIDxtZXRhIG5hbWU9ImFwcGxlLW1vYmlsZS13ZWItYXBwLXRpdGxlIiBjb250ZW50PSJNYWluc2FpbCIgLz4KICAgICAgICA8bGluayByZWw9ImFwcGxlLXRvdWNoLWljb24iIHNpemVzPSIxODB4MTgwIiBocmVmPSIvaW1nL2ljb25zL2FwcGxlLXRvdWNoLWljb24tMTgweDE4MC5wbmciIC8+CiAgICAgICAgPGxpbmsgcmVsPSJtYXNrLWljb24iIGhyZWY9ImltZy9pY29ucy9zYWZhcmktcGlubmVkLXRhYi5zdmciIGNvbG9yPSIjZDUxZjI2IiAvPgogICAgICAgIDxtZXRhIG5hbWU9Im1zYXBwbGljYXRpb24tVGlsZUltYWdlIiBjb250ZW50PSIvaW1nL2ljb25zL21zdGlsZS0xNTB4MTUwLnBuZyIgLz4KICAgICAgICA8bWV0YSBuYW1lPSJtc2FwcGxpY2F0aW9uLVRpbGVDb2xvciIgY29udGVudD0iI2Q1MWYyNiIgLz4KCiAgICAgICAgCiAgICAgIDxzY3JpcHQgdHlwZT0ibW9kdWxlIiBjcm9zc29yaWdpbiBzcmM9Ii9hc3NldHMvaW5kZXgtYjk2OGRkNjUuanMiPjwvc2NyaXB0PgogICAgICA8bGluayByZWw9Im1vZHVsZXByZWxvYWQiIGNyb3Nzb3JpZ2luIGhyZWY9Ii9hc3NldHMvdnVldGlmeS00YzY1YjRjMy5qcyI+CiAgICAgIDxsaW5rIHJlbD0ibW9kdWxlcHJlbG9hZCIgY3Jvc3NvcmlnaW4gaHJlZj0iL2Fzc2V0cy9vdmVybGF5c2Nyb2xsYmFycy00NGQ4N2JjZi5qcyI+CiAgICAgIDxsaW5rIHJlbD0ibW9kdWxlcHJlbG9hZCIgY3Jvc3NvcmlnaW4gaHJlZj0iL2Fzc2V0cy9lY2hhcnRzLTliYzU3MGIwLmpzIj4KICAgICAgPGxpbmsgcmVsPSJzdHlsZXNoZWV0IiBocmVmPSIvYXNzZXRzL3Z1ZXRpZnktOTUwZDFjYjAuY3NzIj4KICAgICAgPGxpbmsgcmVsPSJzdHlsZXNoZWV0IiBocmVmPSIvYXNzZXRzL292ZXJsYXlzY3JvbGxiYXJzLWExNmJjM2QzLmNzcyI+CiAgICAgIDxsaW5rIHJlbD0ic3R5bGVzaGVldCIgaHJlZj0iL2Fzc2V0cy9pbmRleC01NDcwZjNlNC5jc3MiPgogICAgPGxpbmsgcmVsPSJtYW5pZmVzdCIgaHJlZj0iL21hbmlmZXN0LndlYm1hbmlmZXN0Ij4NCjwhLS0gT2N0b0V2ZXJ5d2hlcmUgSW5qZWN0ZWQgVUkgLS0+PHNjcmlwdCBhc3luYyBjcm9zc29yaWdpbiBzcmM9Ii9vZS91aS5mY2NjZDJmZjA0LmpzIj48L3NjcmlwdD48bGluayBjcm9zc29yaWdpbiByZWw9InN0eWxlc2hlZXQiIGhyZWY9Ii9vZS91aS5mY2NjZDJmZjA0LmNzcyI+DQo8L2hlYWQ+CiAgICA8Ym9keSBzdHlsZT0iYmFja2dyb3VuZC1jb2xvcjogIzEyMTIxMiI+CiAgICAgICAgPG5vc2NyaXB0PgogICAgICAgICAgICA8c3Ryb25nPgogICAgICAgICAgICAgICAgV2UncmUgc29ycnkgYnV0IE1haW5zYWlsIGRvZXNuJ3Qgd29yayBwcm9wZXJseSB3aXRob3V0IEphdmFTY3JpcHQgZW5hYmxlZC4gUGxlYXNlIGVuYWJsZSBpdCB0byBjb250aW51ZS4KICAgICAgICAgICAgPC9zdHJvbmc+CiAgICAgICAgPC9ub3NjcmlwdD4KICAgICAgICA8ZGl2IGlkPSJhcHAiPjwvZGl2PgogICAgPC9ib2R5Pgo8L2h0bWw+CnsiY29tbWFuZCJlbmRlcmluZz0hMSx0aGlzLl9wcm9qZWN0aW9uTWF0cml4PW5ldyBOLHRoaXMuX3Bvc3RQcm9jZXNzZXM9bmV3IEFycmF5LHRoaXMuX2FjdGl2ZU1lc2hlcz1uZXcgcXQoMjU2KSx0aGlzLl9nbG9iYWxQb3NpdGlvbj12Llplcm8oKSx0aGlzLl9jb21wdXRlZFZpZXdNYXRyaXg9Ti5JZGVudGl0eSgpLHRoaXMuX2RvTm90Q29tcHV0ZVByb2plY3Rpb25NYXRyaXg9ITEsdGhpcy5fdHJhbnNmb3JtTWF0cml4PU4uWmVybygpLHRoaXMuX3JlZnJlc2hGcnVzdHVtUGxhbmVzPSEwLHRoaXMuX2Fic29sdXRlUm90YXRpb249Z2UuSWRlbnRpdHkoKSx0aGlzLl9pc0NhbWVyYT0hMCx0aGlzLl9pc0xlZnRDYW1lcmE9ITEsdGhpcy5faXNSaWdodENhbWVyYT0hMSx0aGlzLmdldFNjZW5lKCkuYWRkQ2FtZXJhKHRoaXMpLHMmJiF0aGlzLmdldFNjZW5lKCkuYWN0aXZlQ2FtZXJhJiYodGhpcy5nZXRTY2VuZSgpLmFjdGl2ZUNhbWVyYT10aGlzKSx0aGlzLnBvc2l0aW9uPXQsdGhpcy5yZW5kZXJQYXNzSWQ9dGhpcy5nZXRTY2VuZSgpLmdldEVuZ2luZSgpLmNyZWF0ZVJlbmRlclBhc3NJZCgiQ2FtZXJhICIuY29uY2F0KGUpKX1zdG9yZVN0YXRlKCl7cmV0dXJuIHRoaXMuX3N0YXRlU3RvcmVkPSEwLHRoaXMuX3N0b3JlZEZvdj10aGlzLmZvdix0aGlzfV9yZXN0b3JlU3RhdGVWYWx1ZXMoKXtyZXR1cm4gdGhpcy5fc3RhdGVTdG9yZWQ/KHRoaXMuZm92PXRoaXMuX3N0b3JlZEZvdiwhMCk6ITF9cmVzdG9yZVN0YXRlKCl7cmV0dXJuIHRoaXMuX3Jlc3RvcmVTdGF0ZVZhbHVlcygpPyh0aGlzLm9uUmVzdG9yZVN0YXRlT2JzZXJ2YWJsZS5ub3RpZnlPYnNlcnZlcnModGhpcyksITApOiExfWdldENsYXNzTmFtZSgpe3JldHVybiJDYW1lcmEifXRvU3RyaW5nKGUpe2xldCB0PSJOYW1lOiAiK3RoaXMubmFtZTtpZih0Kz0iLCB0eXBlOiAiK3RoaXMuZ2V0Q2xhc3NOYW1lKCksdGhpcy5hbmltYXRpb25zKWZvcihsZXQgaT0wO2k8dGhpcy5hbmltYXRpb25zLmxlbmd0aDtpKyspdCs9IiwgYW5pbWF0aW9uWzBdOiAiK3RoaXMuYW5pbWF0aW9uc1tpXS50b1N0cmluZyhlKTtyZXR1cm4gdH1hcHBseVZlcnRpY2FsQ29ycmVjdGlvbigpe2NvbnN0IGU9dGhpcy5hYnNvbHV0ZVJvdGF0aW9uLnRvRXVsZXJBbmdsZXMoKTt0aGlzLnByb2plY3Rpb25QbGFuZVRpbHQ9dGhpcy5fc2NlbmUudXNlUmlnaHRIYW5kZWRTeXN0ZW0/LWUueDplLnh9Z2V0IGdsb2JhbFBvc2l0aW9uKCl7cmV0dXJuIHRoaXMuX2dsb2JhbFBvc2l0aW9ufWdldEFjdGl2ZU1lc2hlcygpe3JldHVybiB0aGlzLl9hY3RpdmVNZXNoZXN9aXNBY3RpdmVNZXNoKGUpe3JldHVybiB0aGlzLl9hY3RpdmVNZXNoZXMuaW5kZXhPZihlKSE9PS0xfWlzUmVhZHkoZT0hMSl7aWYoZSl7Zm9yKGNvbnN0IHQgb2YgdGhpcy5fcG9zdFByb2Nlc3NlcylpZih0JiYhdC5pc1JlYWR5KCkpcmV0dXJuITF9cmV0dXJuIHN1cGVyLmlzUmVhZHkoZSl9X2luaXRDYWNoZSgpe3N1cGVyLl9pbml0Q2FjaGUoKSx0aGlzLl9jYWNoZS5wb3NpdGlvbj1uZXcgdihOdW1iZXIuTUFYX1ZBTFVFLE51bWlzLmludGVybmFsVmFsdWU9aX19fSxlKTpudWxsKX0sZ2VuU2xpZGVyKHQpe3JldHVybiB0aGlzLmhpZGVTbGlkZXI/bnVsbDoodHx8KHQ9dGhpcy4kY3JlYXRlRWxlbWVudChabSx7cHJvcHM6e2NvbG9yOnRoaXMuc2xpZGVyQ29sb3J9fSkpLHRoaXMuJGNyZWF0ZUVsZW1lbnQoImRpdiIse3N0YXRpY0NsYXNzOiJ2LXRhYnMtc2xpZGVyLXdyYXBwZXIiLHN0eWxlOnRoaXMuc2xpZGVyU3R5bGVzfSxbdF0pKX0sb25SZXNpemUoKXt0aGlzLl9pc0Rlc3Ryb3llZHx8KGNsZWFyVGltZW91dCh0aGlzLnJlc2l6ZVRpbWVvdXQpLHRoaXMucmVzaXplVGltZW91dD13aW5kb3cuc2V0VGltZW91dCh0aGlzLmNhbGxTbGlkZXIsMCkpfSxwYXJzZU5vZGVzKCl7bGV0IHQ9bnVsbCxlPW51bGw7Y29uc3QgaT1bXSxzPVtdLG49dGhpcy4kc2xvdHMuZGVmYXVsdHx8W10scj1uLmxlbmd0aDtmb3IobGV0IGE9MDthPHI7YSsrKXtjb25zdCBvPW5bYV07aWYoby5jb21wb25lbnRPcHRpb25zKXN3aXRjaChvLmNvbXBvbmVudE9wdGlvbnMuQ3Rvci5vcHRpb25zLm5hbWUpe2Nhc2Uidi10YWJzLXNsaWRlciI6ZT1vO2JyZWFrO2Nhc2Uidi10YWJzLWl0ZW1zIjp0PW87YnJlYWs7Y2FzZSJ2LXRhYi1pdGVtIjppLnB1c2gobyk7YnJlYWs7ZGVmYXVsdDpzLnB1c2gobyl9ZWxzZSBzLnB1c2gobyl9cmV0dXJue3RhYjpzLHNsaWRlcjplLGl0ZW1zOnQsaXRlbTppfX19LHJlbmRlcih0KXtjb25zdHt0YWI6ZSxzbGlkZXI6aSxpdGVtczpzLGl0ZW06bn09dGhpcy5wYXJzZU5vZGVzKCk7cmV0dXJuIHQoImRpdiIse3N0YXRpY0NsYXNzOiJ2LXRhYnMiLGNsYXNzOnRoaXMuY2xhc3NlcyxkaXJlY3RpdmVzOlt7bmFtZToicmVzaXplIixtb2RpZmllcnM6e3F1aWV0OiEwfSx2YWx1ZTp0aGlzLm9uUmVzaXplfV19LFt0aGlzLmdlbkJhcihlLGkpLHRoaXMuZ2VuSXRlbXMocyxuKV0pfX0pLFFtPXgoZ3Qsd2UoInRhYnNCYXIiKSxMKSxQZz1RbS5leHRlbmQoKS5leHRlbmQoKS5leHRlbmQoe25hbWU6InYtdGFiIixwcm9wczp7cmlwcGxlOnt0eXBlOltCb29sZWFuLE9iamVjdF0sZGVmYXVsdDohMH0sdGFiVmFsdWU6e3JlcXVpcmVkOiExfX0sZGF0YTooKT0+KHtwcm94eUNsYXNzOiJ2LXRhYi0tYWN0aXZlIn0pLGNvbXB1dGVkOntjbGFzc2VzKCl7cmV0dXJueyJ2LXRhYiI6ITAsLi4uZ3Qub3B0aW9ucy5jb21wdXRlZC5jbGFzc2VzLmNhbGwodGhpcyksInYtdGFiLS1kaXNhYmxlZCI6dGhpcy5kaXNhYmxlZCwuLi50aGlzLmdyb3VwQ2xhc3Nlc319LHZhbHVlKCl7aWYodGhpcy50YWJWYWx1ZSE9bnVsbClyZXR1cm4gdGhpcy50YWJWYWx1ZTtsZXQgdD10aGlzLnRvfHx0aGlzLmhyZWY7cmV0dXJuIHQ9PW51bGw/dDoodGhpcy4kcm91dGVyJiZ0aGlzLnRvPT09T2JqZWN0KHRoaXMudG8pJiYodD10aGlzLiRyb3V0ZXIucmVzb2x2ZSh0aGlzLnRvLHRoaXMuJHJvdXRlLHRoaXMuYXBwZW5kKS5ocmVmKSx0LnJlcGxhY2UoIiMiLCIiKSl9fSxtZXRob2RzOntjbGljayh0KXtpZih0aGlzLmRpc2FibGVkKXt0LnByZXZlbnREZWZhdWx0KCk7Ljk2MSBZMTQyLjg3NyBFLjAwODgxCkcxIFgxNTMuMDUxIFkxNDIuNzczIEUuMDAzNTgKRzEgWDE1My4xNjIgWTE0Mi43NDcgRS4wMDI5NwpHMSBYMTUzLjIyIFkxNDIuNjEyIEUuMDAzODMKRzEgWDE1My4yODkgWTE0Mi41NzggRS4wMDIKRzEgWDE1My4zMDYgWTE0Mi40ODkgRS4wMDIzNgpHMSBYMTUzLjU5IFkxNDIuMDg5IEUuMDEyNzgKRzEgWDE1My45NTcgWTE0MS42OSBFLjAxNDEzCkcxIFgxNTQuMDA3IFkxNDEuNTkzIEUuMDAyODQKRzEgWDE1NC4xMjQgWTE0MS40OTEgRS4wMDQwNApHMSBYMTU0LjE4OCBZMTQxLjM3NiBFLjAwMzQzCkcxIFgxNTQuMjk5IFkxNDEuMzEgRS4wMDMzNgpHMSBYMTU0LjM3OSBZMTQxLjE0NSBFLjAwNDc4CkcxIFgxNTQuNDkzIFkxNDEuMDkxIEUuMDAzMjkKRzEgWDE1NC41NzEgWTE0MC45NCBFLjAwNDQzCkcxIFgxNTQuNjg1IFkxNDAuODc1IEUuMDAzNDIKRzEgWDE1NC43NzggWTE0MC43MjEgRS4wMDQ2OQpHMSBYMTU0Ljg1NSBZMTQwLjY3MSBFLjAwMjM5CkcxIFgxNTQuOTU0IFkxNDAuNTI0IEUuMDA0NjIKRzEgWDE1NS40NTIgWTEzOS45NDUgRS4wMTk5CkcxIFgxNTUuNzM4IFkxMzkuNTE2IEUuMDEzNDMKRzEgWDE1NS43OTcgWTEzOS4zOTIgRS4wMDM1OApHMSBYMTU1LjkxIFkxMzkuMjg2IEUuMDA0MDQKRzEgWDE1Ni4wMDggWTEzOS4xMzkgRS4wMDQ2CkcxIFgxNTYuMjA2IFkxMzguOTg2IEUuMDA2NTIKRzEgWDE1Ni4zMjIgWTEzOC44NjkgRS4wMDQyOQpHMSBYMTU2LjQ2OCBZMTM4LjYyIEUuMDA3NTIKRzEgWDE1Ni41NjQgWTEzOC40OTkgRS4wMDQwMgpHMSBYMTU2LjkwOSBZMTM3Ljg5OSBFLjAxODAzCkcxIFgxNTcuMTc0IFkxMzcuMzkgRS4wMTQ5NQpHMSBYMTU3LjMxNSBZMTM2Ljk3NyBFLjAxMTM3CkcxIFgxNTcuNDUgWTEzNi43ODQgRS4wMDYxNApHMSBYMTU3LjUwOCBZMTM2LjQwOSBFLjAwOTg5CkcxIFgxNTcuNjI0IFkxMzYuMTg0IEUuMDA2NgpHMSBYMTU3LjY3NiBZMTM1Ljc3NSBFLjAxMDc0CkcxIFgxNTcuNzk3IFkxMzUuNTQ1IEUuMDA2NzcKRzEgWDE1Ny44NzUgWTEzNS4xMyBFLjAxMQpHMSBYMTU3Ljk4MyBZMTM0Ljg4IEUuMDA3MQpHMSBYMTU4LjA4MSBZMTM0LjQ3NyBFLjAxMDgxCkcxIFgxNTguMTg1IFkxMzQuMjUxIEUuMDA2NDgKRzEgWDE1OC4yNTcgWTEzMy45NTggRS4wMDc4NgpHMSBYMTU4LjM4OCBZMTMzLjcwMSBFLjAwNzUyCkcxIFgxNTguNDkzIFkxMzMuMzU4IEUuMDA5MzUKRzEgWDE1OC43NyBZMTMyLjgwNyBFLjAxNjA3CkcxIFgxNTguODM5IFkxMzIuNzUyIEUuMDAyMwpHMSBYMTU5LjU1OSBZMTMxLjU0NCBFLjAzNjY0CkcxIFgxNjAuMDE1IFkxMzAuOTExIEUuMDIwMzMKRzEgWDE2MC4xNzYgWTEzMC41NSBFLjAxMDMKRzEgWDE2MC40NTggWTEzMC4xMjkgRS4wMTMyCkcxIFgxNjAuNzk2IFkxMjkuNDU1IEUuMDE5NjUKRzEgWDE2MS4wMyBZMTI4LjkxOSBFLjAxNTI0CkcxIFgxNjEuMzggWTEyNy45IEUuMDI4MDcKRzEgWDE2MS42MjUgWTEyNi42MzkgRS4wMzM0NwpHMSBYMTYxLjcwOSBZMTI1LjkwNyBFLjAxOTIKRzE4CkcxIFg0Ny44OTQgWTE3My42MjYgRS4wMTI4MgpHMSBYNDcuNDQ2IFkxNzQuNTE1IEUuMDI1OTQKRzEgWDQ2Ljk5OCBZMTc1LjI1MiBFLjAyMjQ3CkcxIFg0Ni41NSBZMTc1LjcwNSBFLjAxNjYKRzEgWDQ2LjEwMSBZMTc1Ljk5OSBFLjAxMzk4CkcxIFg0NS4yMDUgWTE3Ni4zNTcgRS4wMjUxNApHMSBYNDQuNzU2IFkxNzYuNDQxIEUuMDExOQpHMSBYNDQuMzA4IFkxNzYuMjQgRS4wMTI3OQpHMSBYNDMuODYgWTE3NS4zNTEgRS4wMjU5NApHMSBYNDMuNDEyIFkxNzQuNjE0IEUuMDIyNDcKRzEgWDQyLjk2MyBZMTc0LjE2MSBFLjAxNjYyCkcxIFg0Mi41MTUgWTE3My44NjcgRS4wMTM5NgpHMSBYNDEuNjE4IFkxNzMuNTA5IEUuMDI1MTYKRzEgWDQxLjE3IFkxNzMuNDI1IEUuMDExODgKRzEgWDQwLjcyMiBZMTczLjYyNiBFLjAxMjc5CkcxIFg0MC4yNzQgWTE3NC41MTUgRS4wMjU5NApHMSBYMzkuODI1IFkxNzUuMjUyIEUuMDIyNDkKRzEgWDM5LjM3NyBZMTc1LjcwNSBFLjAxNjYKRzEgWDM4LjkyOSBZMTc1Ljk5OSBFLjAxMzk2CkcxIFgzOC4wMzIgWTE3Ni4zNTcgRS4wMjUxNgpHMSBYMzcuNTg0IFkxNzYuNDQxIEUuMDExODgKRzEgWDM3LjEzNiBZMTc2LjI0IEUuMDEyNzkKRzEgWDM2LjY4NyBZMTc1LjM1MSBFLjAyNTk1CkcxIFgzNi4yMzkgWTE3NC42MTQgRS4wMjI0NwpHMSBYMzUuNzkxIFkxNzQuMTYxIEUuMDE2NgpHMSBYMzUuMzQyIFkxNzMuODY3IEUuMDEzOTgKRzEgWDM0LjQ0NiBZMTczLjUwOSBFLjAyNTE0Ck03MyBQNTEgUjIzMApHMSBYMzMuOTk4IFkxNzMuNDI1IEUuMDExODgKRzEgWDMzLjU0OSBZMTczLjYyNiBFLjAxMjgyCkcxIFgzMy4xMDEgWTE3NC41MTUgRS4wMjU5NApHMSBYMzIuNjUzIFkxNzUuMjUyIEUuMDIyNDcKRzEgWDMyLjIwNCBZMTc1LjcwNSBFLjAxNjYyCkcxIFgzMS43NTYgWTE3NS45OTkgRS4wMTM5NgpHMSBYMzAuODYgWTE3Ni4zNTcgRS4wMjUxNApHMSBYMzAuNDExIFkxNzYuNDQxIEUuMDExOQpHMSBYMjkuOTYzIFkxNzYuMjQgRS4wMTI3OQpHMSBYMjkuNTE1IFkxNzUuMzUxIEUuMDI1OTQKRzEgWDI5LjA2NiBZMTc0LjYxNCBFLjAyMjQ5CkcxIFgyOC42MTggWTE3NC4xNjEgRS4wMTY2CkcxIFgyOC4xNyBZMTczLjg2NyBFLjAxMzk2CkcxIFgyNy4yNzMgWTE3My41MDkgRS4wMjUxNgpHMSBYMjYuODI1IFkxNzMuNDI1IEUuMDExODgKRzEgWDI2LjM3NyBZMTczLjYyNiBFLjAxMjc5CkcxIFgyNi4wMTkgWTE3NC4zMzYgRS4wMjA3MgpHMSBYMjYuMDE5IFkxNzkuOTg3IEUuMTQ3MjQKRzEgWDI2LjM3NyBZMTc5LjgyNiBFLjAxMDIzCkcxIFgyNi44MjUgWTE3OC45MzcgRS4wMjU5NApHMSBYMjcuMjczIFkxNzguMjAxIEUuMDIyNDUKRzEgWDI3LjcyMiBZMTc3Ljc0NyBFLjAxNjY0CkcxIFgyOC4xNyBZMTc3LjQ1MyBFLjAxMzk2CkcxIFgyOS4wNjYgWTE3Ny4wOTYgRS4wMjUxMwpHMSBYMjkuNTE1IFkxNzcuMDExIEUuMDExOTEKRzEgWDI5Ljk2MyBZMTc3LjIxMyBFLjAxMjgKRzEgWDMwLjQxMSBZMTc4LjEwMSBFLjAyNTkyCkcxIFgzMC44NiBZMTc4LjgzOCBFLn0udGhlbWUtLWRhcmsudi1saXN0IC52LWxpc3QtLWRpc2FibGVke2NvbG9yOiNmZmZmZmY4MH0udGhlbWUtLWRhcmsudi1saXN0IC52LWxpc3QtZ3JvdXAtLWFjdGl2ZTpiZWZvcmUsLnRoZW1lLS1kYXJrLnYtbGlzdCAudi1saXN0LWdyb3VwLS1hY3RpdmU6YWZ0ZXJ7YmFja2dyb3VuZDojZmZmZmZmMWZ9LnYtc2hlZXQudi1saXN0e2JvcmRlci1yYWRpdXM6MH0udi1zaGVldC52LWxpc3Q6bm90KC52LXNoZWV0LS1vdXRsaW5lZCl7Ym94LXNoYWRvdzowIDAgIzAwMDMsMCAwICMwMDAwMDAyNCwwIDAgIzAwMDAwMDFmfS52LXNoZWV0LnYtbGlzdC52LXNoZWV0LS1zaGFwZWR7Ym9yZGVyLXJhZGl1czowfS52LWxpc3R7ZGlzcGxheTpibG9jaztwYWRkaW5nOjhweCAwO3Bvc2l0aW9uOnN0YXRpYzt0cmFuc2l0aW9uOmJveC1zaGFkb3cgLjI4cyBjdWJpYy1iZXppZXIoLjQsMCwuMiwxKX0udi1saXN0LS1kaXNhYmxlZHtwb2ludGVyLWV2ZW50czpub25lfS52LWxpc3QtLWZsYXQgLnYtbGlzdC1pdGVtOmJlZm9yZXtkaXNwbGF5Om5vbmV9LnYtbGlzdC0tZGVuc2UgLnYtc3ViaGVhZGVye2ZvbnQtc2l6ZTouNzVyZW07aGVpZ2h0OjQwcHg7cGFkZGluZzowIDhweH0udi1saXN0LS1uYXYgLnYtbGlzdC1pdGVtOm5vdCg6bGFzdC1jaGlsZCk6bm90KDpvbmx5LWNoaWxkKSwudi1saXN0LS1yb3VuZGVkIC52LWxpc3QtaXRlbTpub3QoOmxhc3QtY2hpbGQpOm5vdCg6b25seS1jaGlsZCl7bWFyZ2luLWJvdHRvbTo4cHh9LnYtbGlzdC0tbmF2LnYtbGlzdC0tZGVuc2UgLnYtbGlzdC1pdGVtOm5vdCg6bGFzdC1jaGlsZCk6bm90KDpvbmx5LWNoaWxkKSwudi1saXN0LS1uYXYgLnYtbGlzdC1pdGVtLS1kZW5zZTpub3QoOmxhc3QtY2hpbGQpOm5vdCg6b25seS1jaGlsZCksLnYtbGlzdC0tcm91bmRlZC52LWxpc3QtLWRlbnNlIC52LWxpc3QtaXRlbTpub3QoOmxhc3QtY2hpbGQpOm5vdCg6b25seS1jaGlsZCksLnYtbGlzdC0tcm91bmRlZCAudi1saXN0LWl0ZW0tLWRlbnNlOm5vdCg6bGFzdC1jaGlsZCk6bm90KDpvbmx5LWNoaWxkKXttYXJnaW4tYm90dG9tOjRweH0udi1saXN0LS1uYXZ7cGFkZGluZy1sZWZ0OjhweDtwYWRkaW5nLXJpZ2h0OjhweH0udi1saXN0LS1uYXYgLnYtbGlzdC1pdGVte3BhZGRpbmc6MCA4cHh9LnYtbGlzdC0tbmF2IC52LWxpc3QtaXRlbSwudi1saXN0LS1uYXYgLnYtbGlzdC1pdGVtOmJlZm9yZXtib3JkZXItcmFkaXVzOjRweH0udi1hcHBsaWNhdGlvbi0taXMtbHRyIC52LWxpc3Qudi1zaGVldC0tc2hhcGVkIC52LWxpc3QtaXRlbSwudi1hcHBsaWNhdGlvbi0taXMtbHRyIC52LWxpc3Qudi1zaGVldC0tc2hhcGVkIC52LWxpc3QtaXRlbTpiZWZvcmUsLnYtYXBwbGljYXRpb24tLWlzLWx0ciAudi1saXN0LnYtc2hlZXQtLXNoYXBlZCAudi1saXN0LWl0ZW0+LnYtcmlwcGxlX19jb250YWluZXJ7Ym9yZGVyLWJvdHRvbS1yaWdodC1yYWRpdXM6MzJweCFpbXBvcnRhbnQ7Ym9yZGVyLXRvcC1yaWdodC1yYWRpdXM6MzJweCFpbXBvcnRhbnR9LnYtYXBwbGljYXRpb24tLWlzLXJ0bCAudi1sWDE5OS4wNTQgWTE0Ni45ODkgRS4wMDM4MwpHMSBYMTk4Ljk3NiBZMTQ3LjE1OSBFLjAwNDg3CkcxIFgxOTguNjc0IFkxNDcuMzgxIEUuMDA5NzcKRzEgWDE5OC41MjQgWTE0Ny40MjcgRS4wMDQwOQpHMSBYMTk4LjIxOCBZMTQ3LjYyMSBFLjAwOTQ0CkcxIFgxOTcuNzE3IFkxNDcuNjg1IEUuMDEzMTYKRzEgWDE5Ny41MzYgWTE0Ny41ODggRS4wMDUzNQpNMjA0IFMxMDAwCkcxIFgxOTcuMTM4IFkxNDcuNjQ0IEYxMDgwMAo7V0lQRV9TVEFSVApHMSBGODY0MApHMSBYMTk3LjM0NSBZMTQ3LjIzMyBFLS4wOTMwOApHMSBYMTk3LjQ0MSBZMTQ2Ljc4MSBFLS4xMDY3CkcxIFgxOTcuNDY1IFkxNDYuNzQ4IEUtLjAwOTQyCkcxIFgxOTcuNDI2IFkxNDYuNTUxIEUtLjA0NjM3CkcxIFgxOTcuNTE2IFkxNDYuMzEzIEUtLjA1ODc1CkcxIFgxOTcuNjQyIFkxNDYuMjQ2IEUtLjAzMjk1CkcxIFgxOTcuNjg2IFkxNDYuMDY1IEUtLjA0MzAxCkcxIFgxOTcuODU2IFkxNDUuODk0IEUtLjA1NTY4CkcxIFgxOTcuOTE2IFkxNDUuNjU4IEUtLjA1NjIzCkcxIFgxOTguMjYxIFkxNDUuNDc0IEUtLjA5MDI4CkcxIFgxOTguNTI5IFkxNDUuNDA0IEUtLjA2Mzk2CkcxIFgxOTguNzE2IFkxNDUuNDIyIEUtLjA0MzM4CkcxIFgxOTguNzYxIFkxNDUuNDgzIEUtLjAxNzUKRzEgWDE5OS4wMTkgWTE0NS40MTYgRS0uMDYxNTUKRzEgWDE5OS4xMDggWTE0NS40MzggRS0uMDIxMTQKO1dJUEVfRU5ECkcxIFoxLjk1IEY3MjAKRzEgWDE4OC40NDggWTE1MC4zMzQgRjEwODAwCkcxIFoxLjU1IEY3MjAKRzEgRS44IEYyMTAwCk0yMDQgUzgwMAo7VFlQRTpQZXJpbWV0ZXIKRzEgRjE1MDAKRzEgWDE4OC41MjEgWTE1MC4yNiBFLjAwMjcxCkcxIFgxODguODg3IFkxNTAuMDk1IEUuMDEwNDYKRzEgWDE4OS4xNDcgWTE1MC4xMDMgRS4wMDY3OApHMSBYMTg5LjU5NSBZMTUwLjM0NCBFLjAxMzI1CkcxIFgxOTAuMTM3IFkxNTAuOTIyIEUuMDIwNjUKRzEgWDE5MC40NjkgWTE1MS4zNTggRS4wMTQyOApHMSBYMTkwLjk3OCBZMTUyLjM1NiBFLjAyOTE5CkcxIFgxOTEuMjA5IFkxNTMuMjMxIEUuMDIzNTgKRzEgWDE5MS4yNjIgWTE1My43NjkgRS4wMTQwOQpHMSBYMTkxLjI0MSBZMTU0LjQ4IEUuMDE4NTMKRzEgWDE5MS4wNTUgWTE1NS4zMTcgRS4wMjIzNApHMSBYMTkwLjgxMSA8IURPQ1RZUEUgaHRtbD4KPGh0bWwgbGFuZz0iZW4iPgogIDxoZWFkPgogICAgPHNjcmlwdD5zZWxmWyJNb25hY29FbnZpcm9ubWVudCJdID0gKGZ1bmN0aW9uIChwYXRocykgewogICAgICAgICAgcmV0dXJuIHsKICAgICAgICAgICAgZ2xvYmFsQVBJOiBmYWxzZSwKICAgICAgICAgICAgZ2V0V29ya2VyVXJsIDogZnVuY3Rpb24gKG1vZHVsZUlkLCBsYWJlbCkgewogICAgICAgICAgICAgIHZhciByZXN1bHQgPSAgcGF0aHNbbGFiZWxdOwogICAgICAgICAgICAgIGlmICgvXigoaHR0cDopfChodHRwczopfChmaWxlOil8KFwvXC8pKS8udGVzdChyZXN1bHQpKSB7CiAgICAgICAgICAgICAgICB2YXIgY3VycmVudFVybCA9IFN0cmluZyh3aW5kb3cubG9jYXRpb24pOwogICA3NTUgRS4wMDk3NQpHMSBYNzAuNTcxIFkxMDcuNjMgRS4wMDM2MQpHMSBYNzEuNSBZMTA3Ljk3NSBFLjAyNTgyCkcxIFg3MS44OTQgWTEwOC4wODMgRS4wMTA2NApHMSBYNzIuMDY2IFkxMDguMTc5IEUuMDA1MTMKRzEgWDcyLjQ5OSBZMTA4LjMgRS4wMTE3MQpHMSBYNzMuMDQgWTEwOC41NTIgRS4wMTU1NQpHMSBYNzMuMzU5IFkxMDguNjUyIEUuMDA4NzEKRzEgWDc0LjQ4NiBZMTA5LjE2OCBFLjAzMjMKRzEgWDc0Ljk4NCBZMTA5LjMwOCBFLjAxMzQ4CkcxIFg3NS4wODcgWTEwOS4zODMgRS4wMDMzMgpHMSBYNzUuNTg0IFkxMDkuNTc2IEUuMDEzODkKRzEgWDc1Ljg0OCBZMTA5LjU5OCBFLjAwNjkKRzEgWDc2LjA4NSBZMTA5LjUxNyBFLjAwNjUzCkcxIFg3Ni4yNTUgWTEwOS4zNiBFLjAwNjAzCkcxIFg3Ni40MDMgWTEwOS4zOTQgRS4wMDM5NgpHMSBYNzcuMzA1IFkxMDkuNzYyIEUuMDI1MzgKRzEgWDc3LjgyMiBZMTEwLjE3NiBFLjAxNzI2CkcxIFg3OC4yNjMgWTExMC4zNjMgRS4wMTI0OApHMSBYNzguNDYgWTExMC41MTIgRS4wMDY0NApHMSBYNzkuMjI5IFkxMTEuMTk1IEUuMDI2OApHMSBYNzkuNzEyIFkxMTEuNzE1IEUuMDE4NDkKRzEgWDgwLjAxNiBZMTEyLjE0NSBFLjAxMzcyCkcxIFg4MC4zNzYgWTExMi42MDIgRS4wMTUxNgpHMSBYODAuNTkxIFkxMTMuMDY1IEUuMDEzMwpHMSBYODAuOTkgWTExMy44MDUgRS4wMjE5MQpHMSBYODEuMjYgWTExNC4yNjMgRS4wMTM4NQpHMSBYODEuNDc0IFkxMTQuNDY4IEUuMDA3NzIKRzEgWDgxLjc5NiBZMTE0LjYxNSBFLjAwOTIyCkcxIFg4MS43MjcgWTExNC44MzQgRS4wMDU5OApHMSBYODEuNzIyIFkxMTUuMTQ3IEUuMDA4MTYKRzEgWDgyLjA4IFkxMTYuMTIxIEUuMDI3MDQKRzEgWDgyLjU4NCBZMTE3LjYxMiBFLjA0MTAxCkcxIFg4Mi42ODcgWTExNy43OTkgRS4wMDU1NgpHMSBYODIuNzgxIFkxMTguMTc0IEUuMDEwMDcKRzEgWDgyLjg3IFkxMTguMzQzIEUuMDA0OTgKRzEgWDgyLjk4IFkxMTguODQ4IEUuMDEzNDcKRzEgWDgzLjA3NiBZMTE5LjA0OCBFLjAwNTc4CkcxIFg4My4yMyBZMTE5LjE5OSBFLjAwNTYyCkcxIFg4My41MDIgWTExOS4zMDQgRS4wMDc2CkcxIFg4My43OTcgWTExOS4yNzIgRS4wMDc3MwpHMSBYODQuMTM2IFkxMTkuMDU0IEUuMDEwNQpHMSBYODQuMjk5IFkxMTguODc3IEUuMDA2MjcKRzEgWDg0LjM5IFkxMTguNTg0IEUuMDA3OTkKRzEgWDg0LjM2NiBZMTE4LjM3NyBFLjAwNTQzCkcxIFg4NC4yMDUgWTExNy44MTYgRS4wMTUyMQpHMSBYODQuMTIgWTExNy42MzcgRS4wMDUxNgpHMSBYODQuMDY2IFkxMTcuMzQ1IEUuMDA3NzQKRzEgWDgzLjc0MiBZMTE2LjQyMSBFLjAyNTUxCkcxIFg4My42ODggWTExNi4wODIgRS4wMDg5NApHMSBYODMuNTE2IFkxMTUuNzI1IEUuMDEwMzMKRzEgWDgzLjI1OSBZMTE1LjA3NiBFLjAxODE5CkcxIFg4Mi45NTcgWTExNC41MTEgRS4wMTY2OQpHMSBYODIuNzQ5IFkxMTQuMjczIEUuMDA4MjQKRzEgWDgyLjUxNyBZMTE0LjIwOSBFLjAwNjI3CkcxIFg4Mi42MTYgWTExMy45ODkgRS4wMDYyOQpHMUg6MC40Mjg5MQpHMSBGMjcwMApHMSBYMTcxLjI3NSBZMTM1LjY5NiBFLjAwNzM1CjtXSURUSDowLjM5ODY0NQpHMSBYMTcxLjE0OSBZMTM1LjkxOCBFLjAwNTgzCk0yMDQgUzEwMDAKO1dJUEVfU1RBUlQKRzEgRjg2NDAKRzEgWDE3MS4yNzUgWTEzNS42OTYgRS0uMDU4OTQKRzEgWDE3MS40MDEgWTEzNS40NzMgRS0uMDU5MTQKRzEgWDE3MS40MTUgWTEzNS40MzQgRS0uMDA5NTcKO1dJUEVfRU5ECkcxIEUtLjY3MjM1IEYyMTAwCkcxIFoxLjggRjcyMApHMSBYMTUzLjc0NSBZMTAwLjQ1NCBGMTA4MDAKRzEgWjEuNCBGNzIwCkcxIEUuOCBGMjEwMApNMjA0IFM4MDAKO1RZUEU6RXh0ZXJuYWwgcGVyaW1ldGVyCjtXSURUSDowLjQ0OTk5OQpHMSBGMTUwMApHMSBYMTUzLjQ3NyBZMTAwLjQ0NiBFLjAwNjk5CkcxIFgxNTMuNDE1IFkxMDAuNDA3IEUuMDAxOTEKRzEgWDE1My4zNjIgWTEwMC4zMDIgRS4wMDMwNgpHMSBYMTUzLjEyMiBZMTAwLjM1IEUuMDA2MzgKRzEgWDE1Mi45NjYgWTEwMC4yNTcgRS4wMDQ3MwpHMSBYMTUyLjg3MSBZOTkuNjU0IEUuMDE1OTEKRzEgWDE1Mi44MjQgWTk5LjQ5NSBFLjAwNDMyCkcxIFgxNTIuNzUyIFk5OS41OSBFLjAwMzExCkcxIFgxNTIuNTE5IFk5OS41OTQgRS4wMDYwNwpHMSBYMTUyLjM3MyBZOTkuNDk2IEUuMDA0NTgKRzEgWDE1Mi4yOTggWTk5LjA0NCBFLjAxMTk0CkcxIFgxNTIuMTE3IFk5OS4wNTcgRS4wMDQ3MwpHMSBYMTUxLjk5NiBZOTguOTMgRS4wMDQ1NwpHMSBYMTUxLjkwNSBZOTguNjExIEUuMDA4NjQKRzEgWDE1MS44ODIgWTk4LjA2MiBFLjAxNDMyCkcxIFgxNTEuNjIxIFk5OC4wNTQgRS4wMDY4CkcxIFgxNTEuNTQzIFk5Ny45MTcgRS4wMDQxMQpHMSBYMTUxLjQ5MyBZOTcuNjY3IEUuMDA2NjQKRzEgWDE1MS4yNjMgWTk3LjA2MiBFLjAxNjg2CkcxIFgxNTAuOTk2IFk5Ni45MDUgRS4wMDgwNwpHMSBYMTUwLjg1MiBZOTYuNTI0IEUuMDEwNjEKRzEgWDE1MC43MTcgWTk2LjM3MSBFLjAwNTMyCkcxIFgxNTAuNDM4IFk5NS43MzkgRS4wMTgKRzEgWDE1MC4yODQgWTk1LjYzMSBFLjAwNDkKRzEgWDE1MC4xNjUgWTk1LjE1NyBFLjAxMjczCkcxIFgxNTAuMDQ3IFk5NS4wNDkgRS4wMDQxNwpHMSBYMTUwLjAwMSBZOTQuNzY4IEUuMDA3NDIKO1dJRFRIOjAuNDkxNjYyCkcxIFgxNTAuMDQ5IFk5NC42NTYgRS4wMDM0OQo7V0lEVEg6MC41MzMzMjQKRzEgWDE1MC4wOTYgWTk0LjU0NCBFLjAwMzgKO1dJRFRIOjAuNTc0OTg3CkcxIFgxNTAuMTQzIFk5NC40MzIgRS4wMDQxMQo7V0lEVEg6MC42MTY2NDkKCkcxIFgxNTAuMTkxIFk5NC4zMiBFLjAwNDQ0CkcxIFgxNTAuOTExIFk5NC4yMjYgRS4wMjY0NwoKO1dJRFRIOjAuNjAyMzA4CgpHMSBYMTUxLjI3MSBZOTQuMTg3IEUuMDEyODcKRzEgWDE1MS44MzUgWTk0LjAzIEUuMDIwODEKCjtXSURUSDowLjU5MTA1MQpHMSBYMTUyLjQ0IFk5My44ODMgRS4wMjE3CjtXSURUSDowLjU4NzU5NApHMSBYMTUzLjIxNCBZOTMuNjU5IEUuMDI3OTEKO1dJRFRIOjAuNTU0MDI2CkcxIFgxNTMuMzY4IFk5My43MDcgRS4wMDUyNQo7WTE1Mi42MTggRS0uMDA5OApHMSBYMTM0LjE2NyBZMTUyLjU0NyBFLS4wMTY5NwpHMSBYMTM0LjIzOCBZMTUyLjU2NiBFLS4wMTY5NwpHMSBYMTM0LjI4NCBZMTUyLjQyMyBFLS4wMzQ2OQpHMSBYMTM0LjMyOSBZMTUyLjI4MSBFLS4wMzQ0CkcxIFgxMzYuNDM0IFkxNTAuMTc3IEUtLjY4NzE3CjtXSVBFX0VORApHMSBaMS42NSBGNzIwCkcxIFgxMzAuODExIFkxNTIuNDY1IEYxMDgwMApHMSBaMS4yNSBGNzIwCkcxIEUuOCBGMjEwMAo7V0lEVEg6MC40NDk5OTkKRzEgRjQ4MDAKRzEgWDEzMC44MTMgWTE1Mi43NyBFLjAwNzk1CkcxIFgxMzAuOTQgWTE1My4yNTUgRS4wMTMwNgpHMSBYMTMxLjE4OCBZMTUzLjcgRS4wMTMyNwpHMSBYMTMxLjQ1NSBZMTUzLjk4IEUuMDEwMDgKRzEgWDEzMS45MjYgWTE1NC4yMDcgRS4wMTM2MgpHMSBYMTMyLjMwOCBZMTU0LjIzNyBFLjAwOTk4CkcxIFgxMzIuMzQ3IFkxNTQuMzAzIEUuMDAyCkcxIFgxMzIuNTQyIFkxNTQuNzg2IEUuMDEzNTcKRzEgWDEzMi43OTUgWTE1NS4yNyBFLjAxNDIzCkcxIFgxMzMuMTk1IFkxNTYuMjA2IEUuMDI2NTIKRzEgWDEzMy4zMzMgWTE1Ni40NDUgRS4wMDcxOQpHMSBYMTMzLjU1OCBZMTU2Ljk3NyBFLjAxNTA1CkcxIFgxMzMuNzQxIFkxNTcuNTAyIEUuMDE0NDkKRzEgWDEzMy45NDcgWTE1OC4wMDUgRS4wMTQxNgpHMSBYMTMzLjc4IFkxNTguMTU1IEUuMDA1ODUKRzEgWDEzMy42MTQgWTE1OC42NjcgRS4wMTQwMgpHMSBYMTMzLjY1NiBZMTU5LjIyNSBFLjAxNDU4CkcxIFgxMzMuNzkgWTE1OS43MTkgRS4wMTMzNApHMSBYMTM0LjAxOCBZMTYwLjI3NCBFLjAxNTYzCkcxIFgxMzEuMzk5IFkxNjIuODkzIEUuMDk2NTEKRzEgWDEzMS40NzggWTE2Mi42OSBFLjAwNTY4CkcxIFgxMzEuNTczIFkxNjIuMDMgRS4wMTczNwpHMSBYMTMxLjY4OCBZMTYxLjY1OSBFLjAxMDEyCkcxIFgxMzEuNzY2IFkxNjAuNTcyIEUuMDI4NApHMSBYMTMxLjg2MiBZMTU5Ljk2MyBFLjAxNjA2CkcxIFgxMzEuODUxIFkxNTkuNDQ2IEUuMDEzNDcKRzEgWDEzMS43NTEgWTE1OC44OTIgRS4wMTQ2NwpHMSBYMTMxLjY3OCBZMTU4LjMxMSBFLjAxNTI2CkcxIFgxMzEuNDc2IFkxNTcuNzI1IEUuMDE2MTUKRzEgWDEzMS40NDEgWTE1Ny4zMDQgRS4wMTEwMQpHMSBYMTMxLjM1MiBZMTU3LjA4NCBFLjAwNjE4CkcxIFgxMzEuMDIzIFkxNTUuOTk5IEUuMDI5NTQKRzEgWDEzMC41NzEgWTE1NC45ODMgRS4wMjg5NwpHMSBYMTMwLjI0OSBZMTU0LjQ3OCBFLjAxNTYxCkcxIFgxMjkuOTUzIFkxNTQuMDcxIEUuMDEzMTEKRzEgWDEyOS42ODYgWTE1My44OTIgRS4wMDgzOApHMSBYMTI5LjY0NyBZMTUzLjY5NCBFLjAwNTI2CkcxIFgxMjkuNTY5IFkxNTMuNDk3IEUuMDA1NTIKRzEgWDEzMC44MSBZMTUyLjI1NiBFLjA0NTczCkcxIFgxMzAuNDgxIFkxNTMuMTg2IEYxMDgwMAo7V0lEVEg6MC40NjYzMzcKRzEgRjQ4MDAKRzEgWDEzMC4wNjQgWTE1My42MDQgRS4wMTU5OQpHMSBYMTMwLjMzOCBZMTUzLjg3IEUuMDEwMzQKO1dJRFRIOjAuNDQ4Njk5CkcxIFgxMzAuNzk2IFkxNTQuNTY1IFkxMDAuODc3IEUuMDMxNDYKRzEgWDE5Ni4wNTIgWTEwMS4zNTYgRS4wMTk5NgpHMSBYMTk1LjQ4NCBZMTAxLjczNCBFLjAxNzc4CkcxIFgxOTQuNDUxIFkxMDIuMzU0IEUuMDMxMzkKRzEgWDE5My4yNTMgWTEwMi45NSBFLjAzNDg2CkcxIFgxOTIuMTY3IFkxMDMuMzg0IEUuMDMwNDcKRzEgWDE5MC44MDcgWTEwMy44NTUgRS4wMzc1CkcxIFgxODkuNjg1IFkxMDQuMjEyIEUuMDMwNjgKRzEgWDE4OS4xNDEgWTEwNC4zMzIgRS4wMTQ1MgpHMSBYMTg4Ljc2OSBZMTA0LjQ5MiBFLjAxMDU1CkcxIFgxODguNDg5IFkxMDQuNTQgRS4wMDc0CkcxIFgxODcuODgxIFkxMDQuNzI3IEUuMDE2NTcKRzEgWDE4Ni45NTIgWTEwNC45NTQgRS4wMjQ5MgpHMSBYMTg2LjU1NSBZMTA1LjEwNCBFLjAxMTA2CkcxIFgxODYuMjU2IFkxMDUuMTQ1IEUuMDA3ODYKRzEgWDE4NS4yMDEgWTEwNS40NjcgRS4wMjg3NApHMSBYMTg0LjY4MiBZMTA1LjU4MyBFLjAxMzg2CkcxIFgxODMuNDM0IFkxMDUuOTQyIEUuMDMzODQKRzEgWDE4Mi45NDUgWTEwNi4xMjMgRS4wMTM1OQpHMSBYMTgyLjMwOCBZMTA2LjMyNyBFLjAxNzQzCkcxIFgxODEuMTYgWTEwNi42MTkgRS4wMzA4NgpHMSBYMTc4LjUwNiBZMTA3LjY1NyBFLjA3NDI1CkcxIFgxNzguMDY5IFkxMDcuOTE5IEUuMDEzMjgKRzEgWDE3Ny45MiBZMTA3Ljk4OCBFLjAwNDI4CkcxIFgxNzcuMzQ4IFkxMDguMzkzIEUuMDE4MjYKRzEgWDE3NS45NDUgWTEwOS4xNjIgRS4wNDE2OQpHMSBYMTc1LjEwMiBZMTA5LjgyOCBFLjAyNzk5CkcxIFgxNzQuNTQgWTExMC4yMzkgRS4wMTgxNApHMSBYMTczLjI1OSBZMTExLjQ4MSBFLjA0NjQ5CkcxIFgxNzIuMzczIFkxMTIuNTU4IEUuMDM2MzQKRzEgWDE3Mi4wMjEgWTExMy4wNTggRS4wMTU5MwpHMSBYMTcxLjc3NyBZMTEzLjM0NiBFLjAwOTg0CkcxIFgxNzEuNjY1IFkxMTMuNTYyIEUuMDA2MzQKRzEgWDE3MS40MTMgWTExMy45NTcgRS4wMTIyMQpHMSBYMTcxLjA5MSBZMTE0Ljk0NyBFLjAyNzEzCkcxIFgxNzAuNzA0IFkxMTUuODI0IEUuMDI0OTgKRzEgWDE3MC42NjIgWTExNS45NTYgRS4wMDM2MQpHMSBYMTcwLjc3MyBZMTE1Ljk2NyBFLjAwMjkxCkcxIFgxNzEuMDY3IFkxMTYuMTI4IEUuMDA4NzMKRzEgWDE3MS4yMjQgWTExNi4zNzcgRS4wMDc2NwpHMSBYMTcxLjI1NCBZMTE2LjYzNiBFLjAwNjc5CkcxIFgxNzEuMTIzIFkxMTcuMDM4IEUuMDExMDIKRzEgWDE3MS41MDUgWTExNy4xMTUgRS4wMTAxNQpHMSBYMTcxLjcyNSBZMTE3LjMyMyBFLjAwNzg5CkcxIFgxNzEuODc2IFkxMTcuNTkxIEUuMDA4MDIKRzEgWDE3MS45MDggWTExOC4wMTggRS4wMTExNgpHMSBYMTcyLjM0NSBZMTE3LjcyNiBFLjAxMzY5CkcxIFgxNzMuMTM1IFkxMTYuOTg1IEUuMDI4MjIKRzEgWDE3My42NDEgWTExNi40MyBFLjAxOTU3CkcxIFgxNzQuNTU3IFkxMTUuNTM4IEUuMDMzMzEKRzEgWDE3NS4zNTYgWTExNC44NTggRS4wMjczNApHMSBYMTc1LjgyIFkxMTQuNTY4IEUuMDE0MjYKRzEgWDE3Ni42OSBZMTE0LjExMiBFLjAyNTU5CkcxIFgxNzcuNDkzIFkxMTEzMS41MjEgRS4wNDAxNQpHMSBYMjAwLjc2MSBZMTMxLjEzNiBFLjAxNTM5CkcxIFgyMDAuNTQ4IFkxMzEuMDE5IEUuMDA2MzMKRzEgWDIwMC44ODEgWTEzMC44NTkgRS4wMDk2MwpHMSBYMjAxLjAyOCBZMTMwLjc1OCBFLjAwNDY1CkcxIFgyMDEuMTI4IFkxMzAuNzkxIEUuMDAyNzQKRzEgWDIwMS40IFkxMzAuODIzIEUuMDA3MTQKRzEgWDIwMS45MTQgWTEzMC43MzcgRS4wMTM1OApHMSBYMjAyLjEyIFkxMzAuNjY3IEUuMDA1NjcKRzEgWDIwMi43MTEgWTEzMC4zNjMgRS4wMTczMgpHMSBYMjAyLjkyNCBZMTMwLjIxOCBFLjAwNjcxCkcxIFgyMDMuNDUzIFkxMjkuNzQ3IEUuMDE4NDYKRzEgWDIwMy44MjYgWTEyOS4yNzkgRS4wMTU1OQpHMSBYMjAzLjg5OSBZMTI5LjMxOSBFLjAwMjE3CkcxIFgyMDQuNzk1IFkxMjkuNTU4IEUuMDI0MTYKRzEgWDIwNS4yNDQgWTEyOS41MTggRS4wMTE3NQpHMSBYMjA1LjY5MiBZMTI5LjE4OCBFLjAxNDUKRzEgWDIwNi41ODkgWTEyNy45MzUgRS4wNDAxNQpHMSBYMjA3LjAzNyBZMTI3LjU1IEUuMDE1MzkKRzEgWDIwNy40ODUgWTEyNy4zMDMgRS4wMTMzMwpHMSBYMjA4LjM4MiBZMTI3LjA2NCBFLjAyNDE5CkcxIFgyMDguODMgWTEyNy4xMDQgRS4wMTE3MgpHMSBYMjA5LjI3OCBZMTI3LjQzNCBFLjAxNDUKRzEgWDIxMC4xNzUgWTEyOC42ODcgRS4wNDAxNQpHMSBYMjEwLjYyMyBZMTI5LjA3MiBFLjAxNTM5CkcxIFgyMTEuMDcxIFkxMjkuMzE5IEUuMDEzMzMKRzEgWDIxMS45NjggWTEyOS41NTggRS4wMjQxOQpHMSBYMjEyLjQxNiBZMTI5LjUxOCBFLjAxMTcyCkcxIFgyMTIuODY1IFkxMjkuMTg4IEUuMDE0NTIKRzEgWDIxMy43NjEgWTEyNy45MzUgRS4wNDAxNApHMSBYMjE0LjIwOSBZMTI3LjU1IEUuMDE1MzkKRzEgWDIxNC42NTggWTEyNy4zMDMgRS4wMTMzNQpHMSBYMjE1LjU1NCBZMTI3LjA2NCBFLjAyNDE2CkcxIFgyMTYuMDAzIFkxMjcuMTA0IEUuMDExNzUKRzEgWDIxNi40NTEgWTEyNy40MzQgRS4wMTQ1CkcxIFgyMTcuMzQ3IFkxMjguNjg3IEUuMDQwMTQKRzEgWDIxNy43OTYgWTEyOS4wNzIgRS4wMTU0MQpHMSBYMjE4LjI0NCBZMTI5LjMxOSBFLjAxMzMzCkcxIFgyMTkuMTQxIFkxMjkuNTU4IEUuMDI0MTkKRzEgWDIxOS41ODkgWTEyOS41MTggRS4wMTE3MgpHMSBYMjIwLjAzNyBZMTI5LjE4OCBFLjAxNDUKRzEgWDIyMC45MzQgWTEyNy45MzUgRS4wNDAxNQpHMSBYMjIxLjM4MiBZMTI3LjU1IEUuMDE1MzkKRzEgWDIyMS44MyBZMTI3LjMwMyBFLjAxMzMzCkcxIFgyMjIuNzI3IFkxMjcuMDY0IEUuMDI0MTkKRzEgWDIyMy4xNzUgWTEyNy4xMDQgRS4wMTE3MgpHMSBYMjIzLjU2MyBZMTI3LjM5IEUuMDEyNTYKRzEgWDIyMy41NjMgWTEyNS41MTcgRS4wNDg4CkcxIFgyMjIuNzI3IFkxMjQuMzQ5IEUuMDM3NDMKRzEgWDIyMi4yNzkgWTEyMy45NjQgRS4wMTUzOQpHMSBYMjIxLjgzIFkxMjMuNzE2IEUuMDEzMzYKRzEgWDIyMC45MzQgWTEyMy40NzggRS4wMjQxNgpHMSBYMjIwLjQ4NSBZMTIzLjUxOCBFLjAxMTc1CkcxIFgyMjAuMDM3IFkxMjMuODQ4IEUuTG9hZGVkIGZpbGUgaXMgbm90IHZhbGlkIFNWRyInKSl9Y2F0Y2goaSl7dChpKX1lbHNlIHQobmV3IEVycm9yKCJFcnJvciBsb2FkaW5nIFNWRyIpKX0sci5vbmVycm9yPXQsci5zZW5kKCl9KSl9fX07ZnVuY3Rpb24gRUsoZSxzKXtjb25zdCB0PWUuZ2V0RWxlbWVudHNCeVRhZ05hbWUoInRpdGxlIik7aWYodC5sZW5ndGgpdFswXS50ZXh0Q29udGVudD1zO2Vsc2V7Y29uc3Qgcj1kb2N1bWVudC5jcmVhdGVFbGVtZW50TlMoImh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiwidGl0bGUiKTtyLnRleHRDb250ZW50PXMsZS5pbnNlcnRCZWZvcmUocixlLmZpcnN0Q2hpbGQpfX1mdW5jdGlvbiBPSyhlKXtpZihlLmdldElzUGVuZGluZylyZXR1cm4gZTtsZXQgcz0hMCx0PWUudGhlbihyPT4ocz0hMSxyKSxyPT57dGhyb3cgcz0hMSxyfSk7cmV0dXJuIHQuZ2V0SXNQZW5kaW5nPWZ1bmN0aW9uKCl7cmV0dXJuIHN9LHR9dmFyIExLPU9iamVjdC5kZWZpbmVQcm9wZXJ0eSxBSz1PYmplY3QuZ2V0T3duUHJvcGVydHlEZXNjcmlwdG9yLEZLPShlLHMsdCxyKT0+e2Zvcih2YXIgaT1yPjE/dm9pZCAwOnI/QUsocyx0KTpzLG49ZS5sZW5ndGgtMSxhO24+PTA7bi0tKShhPWVbbl0pJiYoaT0ocj9hKHMsdCxpKTphKGkpKXx8aSk7cmV0dXJuIHImJmkmJkxLKHMsdCxpKSxpfTtsZXQgTGg9Y2xhc3MgZXh0ZW5kcyBUKEEpe2NvbnN0cnVjdG9yKCl7c3VwZXIoLi4uYXJndW1lbnRzKSx0aGlzLm1kaUFsZXJ0T2N0YWdvbk91dGxpbmU9bXksdGhpcy5tZGlDb250ZW50U2F2ZT11cCx0aGlzLm1kaUZpbGVVcGxvYWQ9UHksdGhpcy5tZGlDbG9zZT1FYSx0aGlzLm1kaUNsb3NlVGhpY2s9cGUsdGhpcy50b3BiYXJIZWlnaHQ9RW4sdGhpcy5zaG93RW1lcmdlbmN5U3RvcERpYWxvZz0hMSx0aGlzLnVwbG9hZFNuYWNrYmFyPXtzdGF0dXM6ITEsZmlsZW5hbWU6IiIscGVyY2VudDowLHNwZWVkOjAsdG90YWw6MCxjYW5jZWxUb2tlblNvdXJjZTpudWxsfSx0aGlzLmZvcm1hdEZpbGVzaXplPUtlfWdldCBnY29kZUlucHV0RmlsZUFjY2VwdCgpe3JldHVybiB0aGlzLmlzSU9TP1tdOnZpfWdldCBuYXZpRHJhd2VyKCl7cmV0dXJuIHRoaXMuJHN0b3JlLnN0YXRlLm5hdmlEcmF3ZXJ9c2V0IG5hdmlEcmF3ZXIoZSl7dGhpcy4kc3RvcmUuZGlzcGF0Y2goInNldE5hdmlEcmF3ZXIiLGUpfWdldCBjdXJyZW50UGFnZSgpe3JldHVybiB0aGlzLiRyb3V0ZS5mdWxsUGF0aH1nZXQgc2F2ZUNvbmZpZ1BlbmRpbmcoKXt2YXIgZSxzO3JldHVybihzPShlPXRoaXMuJHN0b3JlLnN0YXRlLnByaW50ZXIuY29uZmlnZmlsZSk9PW51bGw/dm9pZCAwOmUuc2F2ZV9jb25maWdfcGVuZGluZykhPW51bGw/czohMX1nZXQgaGlkZVNhdmVDb25maWdGb3JCZWRNYXNoKCl7dmFyIGU7cmV0dXJuKGU9dGhpcy4kc3RvcmUuc3RhdGUuZ3VpLnVpU2V0dGluZ3MuaGlkZVNhdmVDb25maWdGb3JCZWRNYXNoKSE9bnVsbD9lOiExfWdldCBzaG93U2F2ZUNvbmZpZ0J1dHRvbigpe3ZhciBzLHQ7aWYoIXRoaXMua2xpcHBlclJlYWR5Rm9yR3VpKXJldHVybiExO2lmKCFQYXRoKyIvIit0aGlzLmNvbnRleHRNZW51Lml0ZW0uZmlsZW5hbWUsZm9yY2U6ITB9LHthY3Rpb246ImZpbGVzL2dldERlbGV0ZURpciJ9KX1kZWxldGVTZWxlY3RlZEZpbGVzKCl7dGhpcy5zZWxlY3RlZEZpbGVzLmZvckVhY2goZT0+e2lmKGUuaXNEaXJlY3RvcnkpdGhpcy4kc29ja2V0LmVtaXQoInNlcnZlci5maWxlcy5kZWxldGVfZGlyZWN0b3J5Iix7cGF0aDp0aGlzLmN1cnJlbnRQYXRoKyIvIitlLmZpbGVuYW1lLGZvcmNlOiEwfSx7YWN0aW9uOiJmaWxlcy9nZXREZWxldGVEaXIifSk7ZWxzZXtjb25zdCBzPWUuZmlsZW5hbWUuc2xpY2UoMCxlLmZpbGVuYW1lLmxhc3RJbmRleE9mKCIuIikpLHQ9ZS5maWxlbmFtZS5zcGxpdCgiLiIpLnBvcCgpO2lmKHRoaXMuJHNvY2tldC5lbWl0KCJzZXJ2ZXIuZmlsZXMuZGVsZXRlX2ZpbGUiLHtwYXRoOnRoaXMuY3VycmVudFBhdGgrIi8iK2UuZmlsZW5hbWV9LHthY3Rpb246ImZpbGVzL2dldERlbGV0ZUZpbGUifSksdCE9PSJtcDQiKXJldHVybjtjb25zdCByPXMrIi5qcGciO3RoaXMuZmlsZXMuZmluZEluZGV4KG49Pm4uZmlsZW5hbWU9PT1yKSE9PS0xJiZ0aGlzLiRzb2NrZXQuZW1pdCgic2VydmVyLmZpbGVzLmRlbGV0ZV9maWxlIix7cGF0aDp0aGlzLmN1cnJlbnRQYXRoKyIvIityfSx7YWN0aW9uOiJmaWxlcy9nZXREZWxldGVGaWxlIn0pfX0pLHRoaXMuc2VsZWN0ZWRGaWxlcz1bXSx0aGlzLmRlbGV0ZVNlbGVjdGVkRGlhbG9nPSExfX07SG09TEgoW0Qoe2NvbXBvbmVudHM6e1BhbmVsOlcsUGF0aE5hdmlnYXRpb246cm59fSldLEhtKTt2YXIgQUg9ZnVuY3Rpb24oKXt2YXIgZT10aGlzLHM9ZS4kY3JlYXRlRWxlbWVudCx0PWUuX3NlbGYuX2N8fHM7cmV0dXJuIHQoImRpdiIsW3QoVyx7YXR0cnM6e3RpdGxlOmUuJHQoIlRpbWVsYXBzZS5UaW1lbGFwc2VGaWxlcyIpLGljb246ZS5tZGlGaWxlRG9jdW1lbnRNdWx0aXBsZU91dGxpbmUsImNhcmQtY2xhc3MiOiJ0aW1lbGFwc2UtZmlsZXMtcGFuZWwifX0sW3QocSxbdCgkLFt0KF8se3N0YXRpY0NsYXNzOiJjb2wtMTIgZC1mbGV4IGFsaWduLWNlbnRlciJ9LFt0KHNlLHtzdGF0aWNTdHlsZTp7Im1heC13aWR0aCI6IjMwMHB4In0sYXR0cnM6eyJhcHBlbmQtaWNvbiI6ZS5tZGlNYWduaWZ5LGxhYmVsOmUuJHQoIlRpbWVsYXBzZS5TZWFyY2giKSwic2luZ2xlLWxpbmUiOiIiLG91dGxpbmVkOiIiLGNsZWFyYWJsZToiIiwiaGlkZS1kZXRhaWxzIjoiIixkZW5zZToiIn0sbW9kZWw6e3ZhbHVlOmUuc2VhcmNoLGNhbGxiYWNrOmZ1bmN0aW9uKHIpe2Uuc2VhcmNoPXJ9LGV4cHJlc3Npb246InNlYXJjaCJ9fSksdChhZSksZS5zZWxlY3RlZEZpbGVzLmxlbmd0aD90KGcse3N0YXRpY0NsYXNzOiJweC0yIG1pbndpZHRoLTAgbWwtMyIsYXR0cnM6e3RpdGxlOmUuJHQoIlRpbWVsYXBzZS5Eb3dubG9hZCIpLGNvbG9yOiJwcmltYXJ5Iixsb2FkaW5nOmUubG9hZGluZ3MuaW5jbHVkZXMoInRpbWVsYXBzZURvd25sb2FkWmlwIil9LG9uOntjbGljazplLmRvd25sb2FkU2VsZWN0ZWRGaWxlc319LFt0KHYsW2UuX2EuZ2V0Q29sb3IoTGkpO24mJmUuYWRkUnVsZShgLm1vbmFjby1lZGl0b3IgLmZpbmQtd2lkZ2V0IHsgYm9yZGVyOiAxcHggc29saWQgJHtufTsgfWApfSk7dmFyIFV2PWZ1bmN0aW9uKGEsZSx0LGkpe3ZhciBuPWFyZ3VtZW50cy5sZW5ndGgscz1uPDM/ZTppPT09bnVsbD9pPU9iamVjdC5nZXRPd25Qcm9wZXJ0eURlc2NyaXB0b3IoZSx0KTppLG87aWYodHlwZW9mIFJlZmxlY3Q9PSJvYmplY3QiJiZ0eXBlb2YgUmVmbGVjdC5kZWNvcmF0ZT09ImZ1bmN0aW9uIilzPVJlZmxlY3QuZGVjb3JhdGUoYSxlLHQsaSk7ZWxzZSBmb3IodmFyIHI9YS5sZW5ndGgtMTtyPj0wO3ItLSkobz1hW3JdKSYmKHM9KG48Mz9vKHMpOm4+Mz9vKGUsdCxzKTpvKGUsdCkpfHxzKTtyZXR1cm4gbj4zJiZzJiZPYmplY3QuZGVmaW5lUHJvcGVydHkoZSx0LHMpLHN9LFd0PWZ1bmN0aW9uKGEsZSl7cmV0dXJuIGZ1bmN0aW9uKHQsaSl7ZSh0LGksYSl9fSxJaDtjb25zdCBGST01MjQyODg7ZnVuY3Rpb24gTGgoYSxlPSJzaW5nbGUiLHQ9ITEpe2lmKCFhLmhhc01vZGVsKCkpcmV0dXJuIG51bGw7Y29uc3QgaT1hLmdldFNlbGVjdGlvbigpO2lmKGU9PT0ic2luZ2xlIiYmaS5zdGFydExpbmVOdW1iZXI9PT1pLmVuZExpbmVOdW1iZXJ8fGU9PT0ibXVsdGlwbGUiKXtpZihpLmlzRW1wdHkoKSl7Y29uc3Qgbj1hLmdldENvbmZpZ3VyZWRXb3JkQXRQb3NpdGlvbihpLmdldFN0YXJ0UG9zaXRpb24oKSk7aWYobiYmdD09PSExKXJldHVybiBuLndvcmR9ZWxzZSBpZihhLmdldE1vZGVsKCkuZ2V0VmFsdWVMZW5ndGhJblJhbmdlKGkpPEZJKXJldHVybiBhLmdldE1vZGVsKCkuZ2V0VmFsdWVJblJhbmdlKGkpfXJldHVybiBudWxsfWxldCBLZT1JaD1jbGFzcyBleHRlbmRzIFJ7Z2V0IGVkaXRvcigpe3JldHVybiB0aGlzLl9lZGl0b3J9c3RhdGljIGdldChlKXtyZXR1cm4gZS5nZXRDb250cmlidXRpb24oSWguSUQpfWNvbnN0cnVjdG9yKGUsdCxpLG4scyl7c3VwZXIoKSx0aGlzLl9lZGl0b3I9ZSx0aGlzLl9maW5kV2lkZ2V0VmlzaWJsZT1BaS5iaW5kVG8odCksdGhpcy5fY29udGV4dEtleVNlcnZpY2U9dCx0aGlzLl9zdG9yYWdlU2VydmljZT1pLHRoaXMuX2NsaXBib2FyZFNlcnZpY2U9bix0aGlzLl9ub3RpZmljYXRpb25TZXJ2aWNlPXMsdGhpcy5fdXBkYXRlSGlzdG9yeURlbGF5ZXI9bmV3IEJuKDUwMCksdGhpcy5fc3RhdGU9dGhpcy5fcmVnaXN0ZXIobmV3IGNJKSx0aGlzLmxvYWRRdWVyeVN0YXRlKCksdGhpcy5fcmVnaXN0ZXIodGhpcy5fc3RhdGUub25GaW5kUmVwbGFjZVN0YXRlQ2hhbmdlKG89PnRoaXMuX29uU3RhdGVDaGFuZ2VkKG8pKSksdGhpcy5fbW9kZWw9bnVsbCx0aGlzLl9yZWdpc3Rlcih0aGlzLl9lZGl0b3Iub25EaWRDaGFuZ2VNb2RlbCgoKT0+e2NvbnN0IG89dGhpcy5fZWRpdG9yLmdldE1vZGVsKCkmJnRoaXMuX3N0YXRlLmlzUmV2ZWFsZWQ7dGhpcy5kaXNwb3NlTW9kZWwoKSx0aGlzLl9zdGF0ZS5jaGFuZ2Uoe3NlYXJjaFNjb3BlOm51bGwsbWF0Y2hDYXNlOnRoaXMuX3N0b3JhZ2VTZXJ2aWNlLmdldEJvb2xlcmVudCI6IHRydWUsCiAgICAgICJkZWZhdWx0IjogdHJ1ZSwKICAgICAgImV4dHJ1ZGVyIjogewogICAgICAgICJjb3VudCI6IDEsCiAgICAgICAgImRlZmF1bHRFeHRydXNpb25MZW5ndGgiOiA1LAogICAgICAgICJub3p6bGVEaWFtZXRlciI6IDAuNCwKICAgICAgICAib2Zmc2V0cyI6IFsKICAgICAgICAgIFsKICAgICAgICAgICAgMC4wLAogICAgICAgICAgICAwLjAKICAgICAgICAgIF0KICAgICAgICBdLAogICAgICAgICJzaGFyZWROb3p6bGUiOiBmYWxzZQogICAgICB9LAogICAgICAiaGVhdGVkQmVkIjogdHJ1ZSwKICAgICAgImhlYXRlZENoYW1iZXIiOiBmYWxzZSwKICAgICAgImlkIjogIl9kZWZhdWx0IiwKICAgICAgIm1vZGVsIjogIk1LMyIsCiAgICAgICJuYW1lIjogIlBydXNhIiwKICAgICAgInJlc291cmNlIjogImh0dHBzOi8vZGV2LW9wLm9jdG9ldmVyeXdoZXJlLmNvbS9hcGkvcHJpbnRlcnByb2ZpbGVzL19kZWZhdWx0IiwKICAgICAgInZvbHVtZSI6IHsKICAgICAgICAiY3VzdG9tX2JveCI6IHsKICAgICAgICAgICJ4X21heCI6IDI1MC4wLAogICAgICAgICAgInhfbWluIjogMC4wLAogICAgICAgICAgInlfbWF4IjogMjEwLjAsCiAgICAgICAgICAieV9taW4iOiAtNC4wLAogICAgICAgICAgInpfbWF4IjogMjEwLjAsCiAgICAgICAgICAiel9taW4iOiAwLjAKICAgICAgICB9LAogICAgICAgICJkZXB0aCI6IDIxMC4wLAogICAgICAgICJmb3JtRmFjdG9yIjogInJlY3Rhbmd1bGFyIiwKICAgICAgICAiaGVpZ2h0IjogMjEwLjAsCiAgICAgICAgIm9yaWdpbiI6ICJsb3dlcmxlZnQiLAogICAgICAgICJ3aWR0aCI6IDI1MC4wCiAgICAgIH0KICAgIH0KICB9Cn0KeyJqc29ucnBjIjoiMi4wIiwibWV0aG9kIjoicHJpbnRlci5pbmZvIiwicGFyYW1zIjp7fSwiaWQiOjcwfXsianNvbnJwYyI6ICIyLjAiLCAibWV0aG9kIjogIm5vdGlmeV9zdGF0dXNfdXBkYXRlIiwgInBhcmFtcyI6IFt7ImhlYXRlcl9iZWQiOiB7InRlbXBlcmF0dXJlIjogMTkuNTd9LCAidG9vbGhlYWQiOiB7ImVzdGltYXRlZF9wcmludF90aW1lIjogMTQ2MzI2LjkyNzAwOTQ4NjF9LCAiZXh0cnVkZXIiOiB7InRlbXBlcmF0dXJlIjogMTkuOTF9fSwgMTQ2MzI0LjkyODA0MzA2MV19eyJqc29ucnBjIjogIjIuMCIsICJtZXRob2QiOiAibm90aWZ5X3Byb2Nfc3RhdF91cGRhdGUiLCAicGFyYW1zIjogW3sibW9vbnJha2VyX3N0YXRzIjogeyJ0aW1lIjogMTcxNzUyMjQ4NC4yNTYwNzU2LCAiY3B1X3VzYWdlIjogMS43NiwgIm1lbW9yeSI6IDM5MzI4LCAibWVtX3VuaXRzIjogImtCIn0sICJjcHVfdGVtcCI6IDMyLjEyOCwgIm5ldHdvcmsiOiB7ImxvIjogeyJyeF9ieXRlcyI6IDEwOTIyNDY2NzksICJ0eF9ieXRlcyI6IDEwOTIyNDY2NzksICJyeF9wYWNrZXRzIjogMTM2ODQ1MSwgInR4X3BhY2tldHMiOiAxMzY4NDUxLCAicnhfZXJycyI6IDAsICJ0eF9lcnJzIjogMCwgInJ4X2Ryb3AiOiAwLCAidHhfZHJvcCI6IDAsICJiYW5kd2lkdGgiOiA4MzE2MjcxLjc4fSwgImV0aDAiOiB7InJ4X2J5dGVzIjogMTk1MzBIYW5kbGVyPWZ1bmN0aW9uKGUsdCl7aWYoaSgiX2NodW5rSGFuZGxlciIsZSksMjAwPT09ZSYmdClmb3IodmFyIG49LTE7O3RoaXMuYnVmZmVyUG9zaXRpb24rPW4rMSl7dmFyIHI9dC5zbGljZSh0aGlzLmJ1ZmZlclBvc2l0aW9uKTtpZigtMT09PShuPXIuaW5kZXhPZigiXG4iKSkpYnJlYWs7dmFyIG89ci5zbGljZSgwLG4pO28mJihpKCJtZXNzYWdlIixvKSx0aGlzLmVtaXQoIm1lc3NhZ2UiLG8pKX19LHMucHJvdG90eXBlLl9jbGVhbnVwPWZ1bmN0aW9uKCl7aSgiX2NsZWFudXAiKSx0aGlzLnJlbW92ZUFsbExpc3RlbmVycygpfSxzLnByb3RvdHlwZS5hYm9ydD1mdW5jdGlvbigpe2koImFib3J0IiksdGhpcy54byYmKHRoaXMueG8uY2xvc2UoKSxpKCJjbG9zZSIpLHRoaXMuZW1pdCgiY2xvc2UiLG51bGwsInVzZXIiKSx0aGlzLnhvPW51bGwpLHRoaXMuX2NsZWFudXAoKX0sdC5leHBvcnRzPXN9LHsiZGVidWciOnZvaWQgMCwiZXZlbnRzIjozLCJpbmhlcml0cyI6NTR9XSwzMzpbZnVuY3Rpb24oZSx0LG4peyhmdW5jdGlvbihzKXsidXNlIHN0cmljdCI7dmFyIGEsbCx1PWUoIi4uLy4uL3V0aWxzL3JhbmRvbSIpLGM9ZSgiLi4vLi4vdXRpbHMvdXJsIiksZj1mdW5jdGlvbigpe307dC5leHBvcnRzPWZ1bmN0aW9uKGUsdCxuKXtmKGUsdCksYXx8KGYoImNyZWF0ZUZvcm0iKSwoYT1zLmRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoImZvcm0iKSkuc3R5bGUuZGlzcGxheT0ibm9uZSIsYS5zdHlsZS5wb3NpdGlvbj0iYWJzb2x1dGUiLGEubWV0aG9kPSJQT1NUIixhLmVuY3R5cGU9ImFwcGxpY2F0aW9uL3gtd3d3LWZvcm0tdXJsZW5jb2RlZCIsYS5hY2NlcHRDaGFyc2V0PSJVVEYtOCIsKGw9cy5kb2N1bWVudC5jcmVhdGVFbGVtZW50KCJ0ZXh0YXJlYSIpKS5uYW1lPSJkIixhLmFwcGVuZENoaWxkKGwpLHMuZG9jdW1lbnQuYm9keS5hcHBlbmRDaGlsZChhKSk7dmFyIHI9ImEiK3Uuc3RyaW5nKDgpO2EudGFyZ2V0PXIsYS5hY3Rpb249Yy5hZGRRdWVyeShjLmFkZFBhdGgoZSwiL2pzb25wX3NlbmQiKSwiaT0iK3IpO3ZhciBvPWZ1bmN0aW9uKHQpe2YoImNyZWF0ZUlmcmFtZSIsdCk7dHJ5e3JldHVybiBzLmRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJzxpZnJhbWUgbmFtZT0iJyt0KyciPicpfWNhdGNoKGUpe3ZhciBuPXMuZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgiaWZyYW1lIik7cmV0dXJuIG4ubmFtZT10LG59fShyKTtvLmlkPXIsby5zdHlsZS5kaXNwbGF5PSJub25lIixhLmFwcGVuZENoaWxkKG8pO3RyeXtsLnZhbHVlPXR9Y2F0Y2goZSl7fWEuc3VibWl0KCk7ZnVuY3Rpb24gaShlKXtmKCJjb21wbGV0ZWQiLHIsZSksby5vbmVycm9yJiYoby5vbnJlYWR5c3RhdGVjaGFuZ2U9by5vbmVycm9yPW8ub25sb2FkPW51bGwsc2V0VGltZW91dChmdW5jdGlvbigpe2YoImNsZWFuaW5nIHVwIixyKSxvLnBhcmVudE5vZGUucmVtb3ZlQ2hpbGQobyksbz1udWxsfSw1MDApLGwudmFsdWU9IiIsbihlKSl9cmV0dXJuIG8ub25lcnJvcj1mdW5jdGlvbigpe2YoIm9uZXJyb3IiLHIpLGkoKX0sby5vbmxvYWQ9ZnVuYy4zNzkzMjM3NX19LCAxNDY1MTkuMzg0MjI3MTg2XX17Impzb25ycGMiOiAiMi4wIiwgIm1ldGhvZCI6ICJub3RpZnlfc3RhdHVzX3VwZGF0ZSIsICJwYXJhbXMiOiBbeyJtY3UiOiB7Imxhc3Rfc3RhdHMiOiB7Im1jdV9hd2FrZSI6IDAuMDAxLCAibWN1X3Rhc2tfYXZnIjogMWUtMDUsICJtY3VfdGFza19zdGRkZXYiOiA3ZS0wNiwgImJ5dGVzX3dyaXRlIjogODkyMzQ4LCAiYnl0ZXNfcmVhZCI6IDE3MzExNjk1LCAiYnl0ZXNfcmV0cmFuc21pdCI6IDksICJieXRlc19pbnZhbGlkIjogMCwgInNlbmRfc2VxIjogMTQ4Njc2LCAicmVjZWl2ZV9zZXEiOiAxNDg2NzYsICJyZXRyYW5zbWl0X3NlcSI6IDIsICJzcnR0IjogMC4wMDMsICJydHR2YXIiOiAwLjAsICJydG8iOiAwLjAyNSwgInJlYWR5X2J5dGVzIjogMCwgInVwY29taW5nX2J5dGVzIjogMCwgImZyZXEiOiA3MTk5ODU3M319LCAic3lzdGVtX3N0YXRzIjogeyJjcHV0aW1lIjogMTAxMS43NjQ5MTMzMzYsICJtZW1hdmFpbCI6IDYwMzM3NjR9LCAidG9vbGhlYWQiOiB7ImVzdGltYXRlZF9wcmludF90aW1lIjogMTQ2MzIwLjQwMzEwNzU5NzIzfSwgImV4dHJ1ZGVyIjogeyJ0ZW1wZXJhdHVyZSI6IDE5Ljk0fX0sIDE0NjMxOC40MDQwMTUxNTddfWFbeyJoaXN0b3J5Ijp7InN0YXRlIjp7InRleHQiOiJPcGVyYXRpb25hbCIsImZsYWdzIjp7Im9wZXJhdGlvbmFsIjp0cnVlLCJwcmludGluZyI6ZmFsc2UsImNhbmNlbGxpbmciOmZhbHNlLCJwYXVzaW5nIjpmYWxzZSwicmVzdW1pbmciOmZhbHNlLCJmaW5pc2hpbmciOmZhbHNlLCJjbG9zZWRPckVycm9yIjpmYWxzZSwiZXJyb3IiOmZhbHNlLCJwYXVzZWQiOmZhbHNlLCJyZWFkeSI6dHJ1ZSwic2RSZWFkeSI6ZmFsc2V9LCJlcnJvciI6IiJ9LCJqb2IiOnsiZmlsZSI6eyJuYW1lIjpudWxsLCJwYXRoIjpudWxsLCJzaXplIjpudWxsLCJvcmlnaW4iOm51bGwsImRhdGUiOm51bGx9LCJlc3RpbWF0ZWRQcmludFRpbWUiOm51bGwsImxhc3RQcmludFRpbWUiOm51bGwsImZpbGFtZW50Ijp7Imxlbmd0aCI6bnVsbCwidm9sdW1lIjpudWxsfSwidXNlciI6bnVsbH0sImN1cnJlbnRaIjpudWxsLCJwcm9ncmVzcyI6eyJjb21wbGV0aW9uIjpudWxsLCJmaWxlcG9zIjpudWxsLCJwcmludFRpbWUiOm51bGwsInByaW50VGltZUxlZnQiOm51bGwsInByaW50VGltZUxlZnRPcmlnaW4iOm51bGx9LCJvZmZzZXRzIjp7fSwicmVzZW5kcyI6eyJjb3VudCI6MCwidHJhbnNtaXR0ZWQiOjgsInJhdGlvIjowfSwidGVtcHMiOlt7InRpbWUiOjE3MTc1MjA4MjYsInRvb2wwIjp7ImFjdHVhbCI6MjAuNywidGFyZ2V0IjowLjB9LCJiZWQiOnsiYWN0dWFsIjoxOC42LCJ0YXJnZXQiOjAuMH0sImNoYW1iZXIiOnsiYWN0dWFsIjpudWxsLCJ0YXJnZXQiOm51bGx9LCJQIjp7ImFjdHVhbCI6MC4wLCJ0YXJnZXQiOm51bGx9LCJBIjp7ImFjdHVhbCI6MjYuNSwidGFyZ2V0IjpudWxsfX0seyJ0aW1lIjoxNzE3NTIwODI4LCJ0b29sMCI6eyJhY3R1YWwiOjIxLjAsInRhcmdldCI6MC4wfSwiYmVkIjp7ImFjdHVhbCI6MTkuMiwidGFyCkcxIFgxMzguMDA0IFk5NC42NjcgRS0uMDI5OTEKRzEgWDEzOC4xNDIgWTk0LjgxNyBFLS4wNDcwNgpHMSBYMTM4LjI4NSBZOTUuMDc5IEUtLjA2ODkyCkcxIFgxMzguNTk0IFk5NS4zNzkgRS0uMDk5NDQKRzEgWDEzOC42MSBZOTUuNDIgRS0uMDEwMTYKRzEgWDEzOC40MzkgWTk1LjQyIEUtLjAzOTQ4CkcxIFgxMzcuOTA0IFk5NS40ODIgRS0uMTI0MzYKRzEgWDEzNy40NDkgWTk1LjQ4IEUtLjEwNTA2CkcxIFgxMzcuMjEgWTk1LjYzOCBFLS4wNjYxNQpHMSBYMTM3LjA4NiBZOTUuODQyIEUtLjA1NTEyCkcxIFgxMzYuOTk5IFk5Ni4xNSBFLS4wNzM5CkcxIFgxMzcuMDM2IFk5Ni4zMDUgRS0uMDM2OApHMSBYMTM3LjE2MSBZOTYuNDA4IEUtLjAzNzQKRzEgWDEzNy4xNTkgWTk2LjQzNSBFLS4wMDYyNAo7V0lQRV9FTkQKRzEgWjEuMzUgRjcyMApHMSBYMTM5LjE0NyBZOTMuODI5IEYxMDgwMApHMSBaLjk1IEY3MjAKRzEgRS44IEYyMTAwCkcxIEY0ODAwCkcxIFgxNDIuNjEgWTkwLjM2NiBFLjEyNzYxCkcxIFgxNDMuMDQgWTkwLjUyNyBFLjAxMTk2CkcxIFgxMzkuODIyIFk5My43NDUgRS4xMTg1OAo7V0lQRV9TVEFSVApHMSBGODY0MApHMSBYMTQyLjI3MiBZOTEuMjk1IEUtLjgKO1dJUEVfRU5ECkcxIFoxLjM1IEY3MjAKRzEgWDE0MC45NyBZOTMuODI2IEYxMDgwMApHMSBaLjk1IEY3MjAKRzEgRS44IEYyMTAwCjtXSURUSDowLjM3MjE5CkcxIEY0ODAwCkcxIFgxNDAuNTgyIFk5My44OTYgRS4wMDgzNgo7V0lEVEg6MC40MTEwOTUKRzEgWDE0MC41MDEgWTkzLjgxNyBFLjAwMjY3CjtXSURUSDowLjQ0OTk5OQpHMSBYMTQwLjQyIFk5My43MzggRS4wMDI5NQpHMSBYMTQzLjU5NCBZOTAuNTY0IEUuMTE2OTYKRzEgWDE0NC4wMzcgWTkwLjYxNCBFLjAxMTYyCkcxIFgxNDQuMDM3IFk5MC43MTIgRS4wMDI1NQpHMSBYMTQyLjU3OSBZOTIuMTcgRS4wNTM3MgpHMSBYMTQyLjczNyBZOTIuMzI5IEUuMDA1ODQKRzEgWDE0Mi43MzMgWTkyLjQyIEUuMDAyMzcKRzEgWDE0NC4wNTkgWTkyLjM0MSBFLjAzNDYxCkcxIFgxNDUuNDEyIFk5Mi4zNjcgRS4wMzUyNgpHMSBYMTQ1Ljc0OCBZOTIuMjc3IEUuMDA5MDYKRzEgWDE0Ni4wNiBZOTIuMTUzIEUuMDA4NzUKO1dJRFRIOjAuNDQ1NDg3CkcxIFgxNDYuMzM2IFk5Mi4yMTQgRS4wMDcyOQo7V0lEVEg6MC40MzA1MjYKRzEgWDE0Ni45NTEgWTkyLjI0NCBFLjAxNTMKRzEgWDE0Ni45ODEgWTkyLjIyNyBFLjAwMDg2CkcxIFgxNDYuODA3IFk5Mi4wMjcgRS4wMDY1OQo7V0lEVEg6MC40NDAwMDEKRzEgWDE0Ni4zMTcgWTkxLjc5MSBFLjAxMzgzCjtXSURUSDowLjQ0OTkxOApHMSBYMTQ2LjE3NCBZOTEuNjQyIEUuMDA1MzgKRzEgWDE0Ni4wMzYgWTkxLjE5MiBFLjAxMjI2CkcxIFgxNDUuODUzIFk5MS4wMTkgRS4wMDY1NgpHMSBYMTQ1LjYyOSBZOTAuOTI3IEUuMDA2MzEKRzEgWDE0NS4wMjMgWTkwLjc0NiBFLjAxNjQ4CkcxIFgxNDQuMjQ1IFk5MC42MzggRS4wMjA0NgpHMSBYMTQ0LjI3OCBZOTEuMDYyIEUuMDExMDgKRzEgWDE0My4zNzMgWTkxLjk2NyBFLjAzMzM0Ckc2NgpHMSBYNjcuMDk2IFkxMjIuNjgyIEUuMDE0NzQKRzEgWDY2LjU1NCBZMTIyLjY2OCBFLjAxNDEzCkcxIFg2Ni4xNTEgWTEyMi42MTkgRS4wMTA1OApHMSBYNjUuOSBZMTIyLjUzNyBFLjAwNjg4CkcxIFg2NS40MzMgWTEyMi4yMzQgRS4wMTQ1CkcxIFg2NC45MDQgWTEyMS45NjEgRS4wMTU1MQpHMSBYNjQuMDI5IFkxMjEuMzQ4IEUuMDI3ODQKRzEgWDYzLjgzIFkxMjEuMTM2IEUuMDA3NTgKRzEgWDYzLjYzOSBZMTIwLjc1NCBFLjAxMTEzCkcxIFg2My42MjkgWTEyMC40NDkgRS4wMDc5NQpHMSBYNjIuNDg1IFkxMTkuOTAzIEUuMDMzMDMKRzEgWDYxLjY2NCBZMTE5LjYxMiBFLjAyMjcKRzEgWDYxLjE1OCBZMTE5LjQwNSBFLjAxNDI0CkcxIFg2MC45MzQgWTExOS4yMzMgRS4wMDczNgpHMSBYNjAuNzg1IFkxMTguODA2IEUuMDExNzgKRzEgWDYwLjgyNiBZMTE4LjY1NiBFLjAwNDA1CkcxIFg2MC40NjMgWTExOC41NzQgRS4wMDk3CkcxIFg1OS41MjggWTExOC40NjkgRS4wMjQ1MgpHMSBYNTguOTI4IFkxMTguMzc2IEUuMDE1ODIKRzEgWDU3LjU5OSBZMTE4LjI4NCBFLjAzNDcxCkcxIFg1Ni45NyBZMTE4LjIxNSBFLjAxNjQ5Ck03MyBRNDQgUzI2NApHMSBYNTYuNDI4IFkxMTguMjI0IEUuMDE0MTIKRzEgWDU1LjcwNCBZMTE4LjE2OSBFLjAxODkyCkcxIFg1NC43MjQgWTExOC4xNjYgRS4wMjU1MwpHMSBYNTMuMzUgWTExOC4wMTEgRS4wMzYwMwpHMSBYNTMuMDY4IFkxMTcuOTA4IEUuMDA3ODIKRzEgWDUyLjgwNSBZMTE3LjcyIEUuMDA4NDIKRzEgWDUyLjUxNiBZMTE3LjU3OSBFLjAwODM4CkcxIFg1Mi4xMTEgWTExNy4yODQgRS4wMTMwNgpHMSBYNTEuNzgzIFkxMTcuMTAzIEUuMDA5NzYKRzEgWDUxLjQyNSBZMTE2Ljc4NyBFLjAxMjQ0CkcxIFg1MS4xMTkgWTExNi4zMiBFLjAxNDU1CkcxIFg1MC44NjkgWTExNi4wMDYgRS4wMTA0NgpHMSBYNTAuNzk1IFkxMTUuODA3IEUuMDA1NTMKRzEgWDUwLjU0NiBZMTE1LjM1NCBFLjAxMzQ3CkcxIFg1MC41NSBZMTE1LjI5MSBFLjAwMTY0CkcxIFg1MC40MTEgWTExNS4xNDggRS4wMDUyCkcxIFg1MC4zMyBZMTE0Ljg5NCBFLjAwNjk1CkcxIFg1MC4zMzUgWTExNC44MDEgRS4wMDI0MwpHMSBYNTAuMDQ2IFkxMTQuNTIzIEUuMDEwNDUKRzEgWDQ5LjY5OSBZMTE0LjMyNSBFLjAxMDQxCkcxIFg0OS4wMTcgWTExMy42ODkgRS4wMjQzCkcxIFg0OC44NDUgWTExMy40NTggRS4wMDc1CkcxIFg0OC43NDIgWTExMy4wNjUgRS4wMTA1OQpHMSBYNDguODE0IFkxMTIuNzc0IEUuMDA3ODEKRzEgWDQ5LjAwNyBZMTEyLjU1NCBFLjAwNzYzCkcxIFg0OS4zMDcgWTExMi40MDQgRS4wMDg3NApHMSBYNDkuNTQ1IFkxMTIuMzk0IEUuMDA2MjEKRzEgWDQ5Ljc0MiBZMTEyLjQ1OCBFLjAwNTQKRzEgWDUwLjEzNSBZMTEyLjc4OCBFLjAxMzM3CkcxIFg1MC42MTggWTExMy4wNDUgRS4wMTQyNgpHMSBYNTAuOTY5IFkxMTMuMzc0IEUuMDEyNTMKRzEgWDUxLjQ0NCBZMTEzLjYzOSBFLjAxNDE3CkcxIFg1MS44NTIgWTExMy44OTUgRS4wMTI1NQpHMSBYNTIuMjIyIFkxMTQuMDQ1IGxvcjojNDJiOTgzO3dvcmQtYnJlYWs6YnJlYWstd29yZDt3aGl0ZS1zcGFjZTpub3JtYWx9Lmp2LWNvbnRhaW5lci5qdi1saWdodCAuanYtaXRlbS5qdi1zdHJpbmcgLmp2LWxpbmt7Y29sb3I6IzAzNjZkNn0uanYtY29udGFpbmVyLmp2LWxpZ2h0IC5qdi1jb2RlIC5qdi10b2dnbGU6YmVmb3Jle3BhZGRpbmc6MHB4IDJweDtib3JkZXItcmFkaXVzOjJweH0uanYtY29udGFpbmVyLmp2LWxpZ2h0IC5qdi1jb2RlIC5qdi10b2dnbGU6aG92ZXI6YmVmb3Jle2JhY2tncm91bmQ6I2VlZX0uanYtY29udGFpbmVyIC5qdi1jb2Rle292ZXJmbG93OmhpZGRlbjtwYWRkaW5nOjMwcHggMjBweH0uanYtY29udGFpbmVyIC5qdi1jb2RlLmJveGVke21heC1oZWlnaHQ6MzAwcHh9Lmp2LWNvbnRhaW5lciAuanYtY29kZS5vcGVue21heC1oZWlnaHQ6aW5pdGlhbCAhaW1wb3J0YW50O292ZXJmbG93OnZpc2libGU7b3ZlcmZsb3cteDphdXRvO3BhZGRpbmctYm90dG9tOjQ1cHh9Lmp2LWNvbnRhaW5lciAuanYtdG9nZ2xle2JhY2tncm91bmQtaW1hZ2U6dXJsKCIrcCtgKTtiYWNrZ3JvdW5kLXJlcGVhdDpuby1yZXBlYXQ7YmFja2dyb3VuZC1zaXplOmNvbnRhaW47YmFja2dyb3VuZC1wb3NpdGlvbjpjZW50ZXIgY2VudGVyO2N1cnNvcjpwb2ludGVyO3dpZHRoOjEwcHg7aGVpZ2h0OjEwcHg7bWFyZ2luLXJpZ2h0OjJweDtkaXNwbGF5OmlubGluZS1ibG9jazstd2Via2l0LXRyYW5zaXRpb246LXdlYmtpdC10cmFuc2Zvcm0gMC4xczt0cmFuc2l0aW9uOi13ZWJraXQtdHJhbnNmb3JtIDAuMXM7dHJhbnNpdGlvbjp0cmFuc2Zvcm0gMC4xczt0cmFuc2l0aW9uOnRyYW5zZm9ybSAwLjFzLCAtd2Via2l0LXRyYW5zZm9ybSAwLjFzfS5qdi1jb250YWluZXIgLmp2LXRvZ2dsZS5vcGVuey13ZWJraXQtdHJhbnNmb3JtOnJvdGF0ZSg5MGRlZyk7dHJhbnNmb3JtOnJvdGF0ZSg5MGRlZyl9Lmp2LWNvbnRhaW5lciAuanYtbW9yZXtwb3NpdGlvbjphYnNvbHV0ZTt6LWluZGV4OjE7Ym90dG9tOjA7bGVmdDowO3JpZ2h0OjA7aGVpZ2h0OjQwcHg7d2lkdGg6MTAwJTt0ZXh0LWFsaWduOmNlbnRlcjtjdXJzb3I6cG9pbnRlcn0uanYtY29udGFpbmVyIC5qdi1tb3JlIC5qdi10b2dnbGV7cG9zaXRpb246cmVsYXRpdmU7dG9wOjQwJTt6LWluZGV4OjI7Y29sb3I6Izg4ODstd2Via2l0LXRyYW5zaXRpb246YWxsIDAuMXM7dHJhbnNpdGlvbjphbGwgMC4xczstd2Via2l0LXRyYW5zZm9ybTpyb3RhdGUoOTBkZWcpO3RyYW5zZm9ybTpyb3RhdGUoOTBkZWcpfS5qdi1jb250YWluZXIgLmp2LW1vcmUgLmp2LXRvZ2dsZS5vcGVuey13ZWJraXQtdHJhbnNmb3JtOnJvdGF0ZSgtOTBkZWcpO3RyYW5zZm9ybTpyb3RhdGUoLTkwZGVnKX0uanYtY29udGFpbmVyIC5qdi1tb3JlOmFmdGVye2NvbnRlbnQ6IiI7d2lkdGg6MTAwJTtoZWlnaHQ6MTAwJTtwb3NpdGlvbjphYnNvbHV0ZTtib3R0b206MDtsZWZ0OjA7ei1pbmRleDoxO2JhY2tncm91bmQ6LXdlYmtpdC1saW5lYXItZ3JhZGllbnQodG9wLCByZ2JhKDAsMCwwLDApIDIwJSwgcmdiYSgyMzAsMjMwLHRoaXMuJHN0b3JlLmRpc3BhdGNoKCJjb25maWcvdXBkYXRlSGVhZGVyIix7bmFtZTp0aGlzLmtleU5hbWUsaGVhZGVyOmV9KX19O0dtKFt5KHt0eXBlOlN0cmluZyxyZXF1aXJlZDohMH0pXSxubC5wcm90b3R5cGUsImtleU5hbWUiLDIpO0dtKFt5KHt0eXBlOkFycmF5LHJlcXVpcmVkOiEwfSldLG5sLnByb3RvdHlwZSwiaGVhZGVycyIsMik7R20oW3koe3R5cGU6Qm9vbGVhbn0pXSxubC5wcm90b3R5cGUsImRpc2FibGVkIiwyKTtubD1HbShbUCh7fSldLG5sKTt2YXIgQ1k9ZnVuY3Rpb24oKXt2YXIgZT10aGlzLHQ9ZS5fc2VsZi5fYztyZXR1cm4gZS5fc2VsZi5fc2V0dXBQcm94eSx0KGt0LHthdHRyczp7Ym90dG9tOiIiLGxlZnQ6IiIsIm9mZnNldC15IjoiIix0cmFuc2l0aW9uOiJzbGlkZS15LXRyYW5zaXRpb24iLCJtaW4td2lkdGgiOiIxNTAiLCJjbG9zZS1vbi1jb250ZW50LWNsaWNrIjohMX0sc2NvcGVkU2xvdHM6ZS5fdShbe2tleToiYWN0aXZhdG9yIixmbjpmdW5jdGlvbih7b246cyxhdHRyczpufSl7cmV0dXJuW3QoTGUse2F0dHJzOntib3R0b206IiJ9LHNjb3BlZFNsb3RzOmUuX3UoW3trZXk6ImFjdGl2YXRvciIsZm46ZnVuY3Rpb24oe29uOml9KXtyZXR1cm5bdChOZSxlLl9nKGUuX2Ioe2F0dHJzOntkaXNhYmxlZDplLmRpc2FibGVkLGZhYjoiIixzbWFsbDoiIix0ZXh0OiIifX0sInYtYnRuIixuLCExKSx7Li4uaSwuLi5zfSksW3QoeCxbZS5fdigiICR0YWJsZUNvbHVtbiAiKV0pXSwxKV19fV0sbnVsbCwhMCl9LFt0KCJzcGFuIixbZS5fdihlLl9zKGUuJHQoImFwcC5nZW5lcmFsLmJ0bi5zZWxlY3RfY29sdW1ucyIpKSldKV0pXX19XSl9LFt0KER0LHtzdGF0aWNDbGFzczoib3ZlcmZsb3cteS1hdXRvIixhdHRyczp7ZGVuc2U6IiJ9fSxbZS5fbChlLmhlYWRlcnMsZnVuY3Rpb24ocyl7cmV0dXJuW3MudGV4dCE9PSIiJiZzLmNvbmZpZ3VyYWJsZT90KHdlLHtrZXk6cy52YWx1ZSxvbjp7Y2xpY2s6ZnVuY3Rpb24obil7cmV0dXJuIGUuaGFuZGxlVG9nZ2xlSGVhZGVyKHMpfX19LFt0KGlyLHtzdGF0aWNDbGFzczoibXktMCJ9LFt0KGFyLHthdHRyczp7ImlucHV0LXZhbHVlIjpzLnZpc2libGV9fSldLDEpLHQoRWUsW3QoU2UsW2UuX3YoZS5fcyhzLnRleHQpKV0pXSwxKV0sMSk6ZS5fZSgpXX0pXSwyKV0sMSl9LFBZPVtdLEFZPUUobmwsQ1ksUFksITEsbnVsbCxudWxsLG51bGwsbnVsbCk7Y29uc3QgcW09QVkuZXhwb3J0czt2YXIgRVk9T2JqZWN0LmRlZmluZVByb3BlcnR5LExZPU9iamVjdC5nZXRPd25Qcm9wZXJ0eURlc2NyaXB0b3IsRnQ9KHIsZSx0LHMpPT57Zm9yKHZhciBuPXM+MT92b2lkIDA6cz9MWShlLHQpOmUsaT1yLmxlbmd0aC0xLGE7aT49MDtpLS0pKGE9cltpXSkmJihuPShzP2EoZSx0LG4pOmEobikpfHxuKTtyZXR1cm4gcyYmbiYmRVkoZSx0LG4pLG59O2xldCBBdD1jbGFzcyBleHRlbmRzIEcobHQpe2NvbnN0cnVjdG9yKCl7c3VwZXIoLi4uYXJndW1lbnRzKTtmKHRoaXMsImlucHV0VmFsdWUiKTtmKHRoaXMsInJlc2V0VmFsdWUiKTtmKHRoaXMsImxhYmVsIikgWDEyOS41OTMgWTEzMi43NyBFLjAxNDk5CkcxIFgxMjkuMTUgWTEzMy40NTUgRS4wMjEyNgpHMSBYMTI4LjU3NiBZMTM0LjQ1NCBFLjAzMDAyCkcxIFgxMjguMjM4IFkxMzQuOTY2IEUuMDE1OTkKRzEgWDEyNy43NjcgWTEzNS42NDkgRS4wMjE2MgpHMSBYMTI3LjM0NSBZMTM2LjE0NCBFLjAxNjk1CkcxIFgxMjcuMDA5IFkxMzYuNDQ0IEUuMDExNzQKRzEgWDEyNi4zNzEgWTEzNi45MDYgRS4wMjA1MgpHMSBYMTI2LjE1NiBZMTM2Ljg0NyBFLjAwNTgxCkcxIFgxMjUuODM0IFkxMzYuODUxIEUuMDA4MzkKRzEgWDEyNS42NSBZMTM2LjkzNiBFLjAwNTI4CkcxIFgxMjUuMzI0IFkxMzcuMTc4IEUuMDEwNTgKRzEgWDEyNS4xOTggWTEzNy4zODUgRS4wMDYzMQpHMSBYMTI1LjA1NiBZMTM3Ljg5NiBFLjAxMzgyCkcxIFgxMjUuMDI4IFkxMzguNDEgRS4wMTM0MQpHMSBYMTI1LjA2OCBZMTQwLjY3IEUuMDU4OQpHMSBYMTI0Ljk5NCBZMTQxLjY4MiBFLjAyNjQ0CkcxIFgxMjQuODY4IFkxNDIuMTAxIEUuMDExNApHMSBYMTI0LjgwNSBZMTQyLjQ4MiBFLjAxMDA2CkcxIFgxMjQuNTIzIFkxNDMuMjQgRS4wMjEwNwpHMSBYMTI0LjQ0MSBZMTQzLjYzNiBFLjAxMDU0CkcxIFgxMjQuMzU5IFkxNDMuNzY2IEUuMDA0CkcxIFgxMjQuMjI2IFkxNDQuMTcxIEUuMDExMTEKRzEgWDEyNC4xMTYgWTE0NC4zNjggRS4wMDU4OApHMSBYMTI0LjAwNSBZMTQ0LjcyMyBFLjAwOTY5CkcxIFgxMjMuNzUzIFkxNDUuMTM4IEUuMDEyNjUKRzEgWDEyMy42MjcgWTE0NS40ODYgRS4wMDk2NApHMSBYMTIzIFkxNDYuMTc2IEUuMDI0MjkKRzEgWDEyMS45NDYgWTE0Ny4xOTIgRS4wMzgxNApHMSBYMTIxLjA5NCBZMTQ4LjI5MSBFLjAzNjIzCkcxIFgxMjAuODg2IFkxNDguNjA2IEUuMDA5ODQKRzEgWDEyMC43MDIgWTE0OC43OSBFLjAwNjc4CkcxIFgxMjAuMjUgWTE0OS4zMjcgRS4wMTgyOQpHMSBYMTE5LjkzMiBZMTQ5LjI4OCBFLjAwODM1CkcxIFgxMTkuNjUyIFkxNDkuNCBFLjAwNzg2CkcxIFgxMTkuMjQ4IFkxNDkuNzEgRS4wMTMyNwpHMSBYMTE4LjY0NSBZMTUwLjgxNiBFLjAzMjgyCkcxIFgxMTguMTY0IFkxNTEuNDQ4IEUuMDIwNjkKRzEgWDExNy43OTQgWTE1MS44NDQgRS4wMTQxMgpHMSBYMTE3LjY2NiBZMTUyLjA2OSBFLjAwNjc0CkcxIFgxMTcuNTMgWTE1Mi4xMzIgRS4wMDM5MQpHMSBYMTE2LjgwOCBZMTUxLjkzMSBFLjAxOTUzCkcxIFgxMTYuMTgxIFkxNTEuODcxIEUuMDE2NDEKRzEgWDExNS42OCBZMTUxLjg3NSBFLjAxMzA1CkcxIFgxMTUuMTM5IFkxNTEuNjkyIEUuMDE0ODgKRzEgWDExNC42NjQgWTE1MS42MjEgRS4wMTI1MQpHMSBYMTEzLjg4NiBZMTUxLjY0OSBFLjAyMDI4CkcxIFgxMTMuNDMgWTE1MS42NDEgRS4wMTE4OApHMSBYMTEyLjc3NSBZMTUxLjY3MSBFLjAxNzA4CkcxIFgxMTIuMjczIFkxNTEuNjEgRS4wMTMxOApHMSBYMTEyLjIzIFkxNTEuNTQyIEUuMDAyMQpHMSBYMTEyLjA3NSBZMTUxLjQ3NyBFLjAwNDM4CkcxIFgxMTEuOTI2IFkxNTEuMTQ1IEUuMDA5NDgKRzEgWDExMS42MzEgWTE1MC44MiBFLiBYMTY1LjUxNCBZMTU2LjM0MSBFLjAxMDI1CkcxIFgxNjUuMTkgWTE1Ny4wODEgRS4wMjEwNQpHMSBYMTY0LjYyNCBZMTU3Ljg4NyBFLjAyNTY2CkcxIFgxNjQuMjU4IFkxNTguMjgzIEUuMDE0MDUKRzEgWDE2NC4xNjEgWTE1OC40NDEgRS4wMDQ4MwpHMSBYMTYzLjgyNyBZMTU4Ljg2OCBFLjAxNDEzCkcxIFgxNjMuMTM4IFkxNTkuNjQ4IEUuMDI3MTIKRzEgWDE2Mi40MDggWTE2MC4xNCBFLjAyMjk0CkcxIFgxNjIuMTUxIFkxNjAuMjc3IEUuMDA3NTkKRzEgWDE2MS45MTQgWTE2MC40ODcgRS4wMDgyNQpHMSBYMTYxLjM1NSBZMTYwLjg2MSBFLjAxNzUyCkcxIFgxNjAuNjUzIFkxNjEuMTgxIEUuMDIwMQpHMSBYMTU5Ljg5OCBZMTYxLjQ3NCBFLjAyMTEKRzEgWDE1OC41MDkgWTE2MS44NzIgRS4wMzc2NQpHMSBYMTU4LjUwOCBZMTYyLjA2NyBFLjAwNTA4CkcxIFgxNTguMzk2IFkxNjIuNDA2IEUuMDA5MwpHMSBYMTU4LjI3MSBZMTYyLjU0IEUuMDA0NzcKRzEgWDE1Ny45ODYgWTE2Mi42NjcgRS4wMDgxMwpHMSBYMTU3LjY2MyBZMTYyLjcyOCBFLjAwODU2CkcxIFgxNTcuNDA4IFkxNjIuODI5IEUuMDA3MTUKRzEgWDE1Ni44ODUgWTE2Mi45MTIgRS4wMTM4CkcxIFgxNTYuNTYgWTE2My4wNSBFLjAwOTIKRzEgWDE1NS43NzMgWTE2My4yMiBFLjAyMDk4CkcxIFgxNTQuOTg4IFkxNjMuNDUxIEUuMDIxMzIKRzEgWDE1NC44NSBZMTYzLjU4NSBFLjAwNTAxCkcxIFgxNTQuNzIyIFkxNjMuODIzIEUuMDA3MDQKRzEgWDE1NC40NDcgWTE2NC4xMTMgRS4wMTA0MQpHMSBYMTU0LjE3NiBZMTY0LjIzMiBFLjAwNzcxCkcxIFgxNTEuNjkxIFkxNjQuMzE0IEUuMDY0NzgKRzEgWDE0OS45NTMgWTE2NC4yNDEgRS4wNDUzMgpHMSBYMTQ5LjI0NiBZMTY0LjA4NSBFLjAxODg2CkcxIFgxNDguNzIxIFkxNjQuMDM5IEUuMDEzNzMKRzEgWDE0OC4xOTUgWTE2My44NzYgRS4wMTQzNQpHMSBYMTQ3LjczNyBZMTYzLjc4NSBFLjAxMjE3CkcxIFgxNDcuNTMxIFkxNjMuNjkyIEUuMDA1ODkKRzEgWDE0Ny4wNDIgWTE2My41OCBFLjAxMzA3CkcxIFgxNDYuMTAyIFkxNjMuMjA2IEUuMDI2MzYKRzEgWDE0NC45NjcgWTE2Mi42ODggRS4wMzI1MQpHMSBYMTQ0LjM5NCBZMTYyLjM3OCBFLjAxNjk3CkcxIFgxNDMuOTc5IFkxNjIuMTkzIEUuMDExODQKRzEgWDE0My40NjcgWTE2MS44NDYgRS4wMTYxMgpHMSBYMTQyLjk1NSBZMTYxLjU0NCBFLjAxNTQ5CkcxIFgxNDIuOTY2IFkxNjEuNjUyIEUuMDAyODMKRzEgWDE0Mi44NDYgWTE2MS45NjkgRS4wMDg4MwpHMSBYMTQyLjUxMiBZMTYyLjI2MiBFLjAxMTU4CkcxIFgxNDIuMjI5IFkxNjIuMzQzIEUuMDA3NjcKRzEgWDE0MS45NzEgWTE2Mi4yOTkgRS4wMDY4MgpHMSBYMTQxLjUgWTE2Mi4wOTggRS4wMTMzNApHMSBYMTQxLjE4MyBZMTYxLjg2MiBFLjAxMDMKRzEgWDE0MC42OTcgWTE2MS41OCBFLjAxNDY0CkcxIFgxMzkuNDQ1IFkxNjAuNzk0IEUuMDM4NTIKRzEgWDEzOS45MjQgWTE2MS41NTggRS4wMjM1CkcxIFgxMzkuOTYzIFkxNjEuOTAyIEUuMDA5MDIKRzEgWDEzOS44MzQgWTE2WDEwOS40NTkgWTE3Mi4zMDEgRS4wMTIyOQpHMSBYMTA5LjE4NiBZMTczLjYxNiBFLjAzNDk5CkcxIFgxMDkuMTAxIFkxNzMuODMxIEUuMDA2MDIKRzEgWDEwOS4wMTggWTE3NC4yIEUuMDA5ODUKRzEgWDEwOC45MjggWTE3NC4zNDkgRS4wMDQ1NApHMSBYMTA4Ljc4OCBZMTc0Ljc5NSBFLjAxMjE4CkcxIFgxMDguNTQ2IFkxNzUuMjc0IEUuMDEzOTgKRzEgWDEwOC40NjIgWTE3NS42MTkgRS4wMDkyNQpHMSBYMTA3LjkzMSBZMTc2LjY2MyBFLjAzMDUyCkcxIFgxMDcuNzIyIFkxNzcuMTQ1IEUuMDEzNjkKRzEgWDEwNy4zMzQgWTE3Ny45MTMgRS4wMjI0MgpHMSBYMTA3LjA2NyBZMTc4LjM0MSBFLjAxMzE0CkcxIFgxMDYuODQgWTE3OC43ODkgRS4wMTMwOQpHMSBYMTA2LjU3IFkxNzkuMjM0IEUuMDEzNTYKRzEgWDEwNi4xNzMgWTE3OS45NyBFLjAyMTc5CkcxIFgxMDUuODI3IFkxODAuNDA5IEUuMDE0NTYKRzEgWDEwNS4yNzEgWTE4MS4zMTIgRS4wMjc2MwpHMSBYMTA0LjUxNSBZMTgyLjI0MyBFLjAzMTI1CkcxIFgxMDQuMTQgWTE4Mi42NTEgRS4wMTQ0NApHMSBYMTAzLjQwMSBZMTgzLjM2MSBFLjAyNjcKRzEgWDEwMi40MDcgWTE4NC4xNTEgRS4wMzMwOApHMSBYMTAxLjk3MiBZMTg0LjQzOCBFLjAxMzU4CkcxIFgxMDEuNTMgWTE4NC42OTMgRS4wMTMzCkcxIFgxMDAuNzc3IFkxODUuMDcxIEUuMDIxOTUKRzEgWDEwMC4yNjUgWTE4NS4yODcgRS4wMTQ0OApHMSBYOTkuNzc1IFkxODUuMzcxIEUuMDEyOTUKRzEgWDk5LjQ4MSBZMTg1LjUxOSBFLjAwODU4CjtXSURUSDowLjQ3Nzc3NApHMSBYOTkuMjMyIFkxODUuNTY0IEUuMDA3MDMKO1dJRFRIOjAuNTA1NTQ4CkcxIFg5OC45ODIgWTE4NS42MSBFLjAwNzUKO1dJRFRIOjAuNTMwNjc0CkcxIFg5OC45MiBZMTg1LjYzNyBFLjAwMjEKO1dJRFRIOjAuNTU1OApHMSBYOTguODU3IFkxODUuNjY1IEUuMDAyMjUKO1dJRFRIOjAuNTk1OTQyCkcxIFg5OC43MDQgWTE4NS42NjYgRS4wMDUzOAo7V0lEVEg6MC42MzYwODMKRzEgWDk4LjU1MSBZMTg1LjY2NyBFLjAwNTc2CkcxIFg5OC4yMDEgWTE4NS43ODggRS4wMTM5NQo7V0lEVEg6MC42MDYxMzYKRzEgWDk3LjkzOSBZMTg1Ljg1MyBFLjAwOTY2CjtXSURUSDowLjU3OTkzMgpHMSBYOTcuNjc2IFkxODUuOTE3IEUuMDA5MjUKO1dJRFRIOjAuNTUzNzI4CkcxIFg5Ny4zOCBZMTg2LjA0MSBFLjAxMDQ0CjtXSURUSDowLjU0NjgyOQpHMSBYOTYuOTk2IFkxODYuMTE5IEUuMDEyNTgKO1dJRFRIOjAuNTA3OTg3CkcxIFg5Ni42NSBZMTg2LjIxMSBFLjAxMDYyCjtXSURUSDowLjQ5MzA3MgpHMSBYOTYuMjAyIFkxODYuMzM3IEUuMDEzMzgKO1dJRFRIOjAuNDg1NDg0CkcxIFg5NS44MzQgWTE4Ni40NzQgRS4wMTExCjtXSURUSDowLjQ2NTQ0OApHMSBYOTUuMzg0IFkxODYuNjAxIEUuMDEyNjMKO1dJRFRIOjAuNDU5MzkzCkcxIFg5NS4xNjkgWTE4Ni43MDYgRS4wMDYzNwo7V0lEVEg6MC40ODA2MjIKRzEgWDk0LjgwMiBZMTg3LjA3NiBFLjAxNDU3CkcxIFg5NC44NTIgWTE4Ni43OTkgRS4wMDc4Nwo7V0lEVEg6MC40NjQ3NC40MzggWTExMi44MzggRS0uMDQ0NTgKRzEgWDE3NC4zMzIgWTExMi45OTkgRS0uMDQ0NTEKRzEgWDE3NC4yMjYgWTExMy4xNjEgRS0uMDQ0Nwo7V0lQRV9FTkQKRzEgRS0uNDU3NzggRjIxMDAKRzEgWjEuNSBGNzIwCkcxIFgxNzUuNTYxIFkxMDkuNTMgRjEwODAwCkcxIFoxLjEgRjcyMApHMSBFLjggRjIxMDAKO1dJRFRIOjAuNDQ5OTk5CkcxIEY0ODAwCkcxIFgxNzguOTkxIFkxMTIuOTYxIEUuMTI2NDEKRzEgWDE3OS4yNjcgWTExMi42NDUgRS4wMTA5MwpHMSBYMTc1Ljg5NiBZMTA5LjI3NCBFLjEyNDIyCkcxIFgxNzYuMjUyIFkxMDkuMDQgRS4wMTExCkcxIFgxNzkuNjIgWTExMi40MDggRS4xMjQxMQpHMSBYMTgwLjA0IFkxMTIuMjM3IEUuMDExODIKRzEgWDE3Ni42NDIgWTEwOC44MzkgRS4xMjUyMQpHMSBYMTc3LjA0MSBZMTA4LjY0NyBFLjAxMTU0CkcxIFgxODAuNDg3IFkxMTIuMDk0IEUuMTI3CkcxIFgxODAuOTIzIFkxMTEuOTM4IEUuMDEyMDcKRzEgWDE3Ny40MDYgWTEwOC40MjEgRS4xMjk2CkcxIFgxNzcuNzggWTEwOC4yMDUgRS4wMTEyNQpHMSBYMTgxLjM5OSBZMTExLjgyNCBFLjEzMzM1CkcxIFgxODEuODY1IFkxMTEuNjk4IEUuMDEyNTgKRzEgWDE3OC4xOTMgWTEwOC4wMjcgRS4xMzUyOQo7V0lQRV9TVEFSVApHMSBGODY0MApHMSBYMTgwLjY0MyBZMTEwLjQ3NyBFLS44CjtXSVBFX0VORApHMSBaMS41IEY3MjAKRzEgWDE2NS41OSBZMTE1Ljc5MiBGMTA4MDAKRzEgWjEuMSBGNzIwCkcxIEUuOCBGMjEwMApHMSBGNDgwMApHMSBYMTY1LjgyOSBZMTE0Ljk2OCBFLjAyMjM1CkcxIFgxNjUuOTg2IFkxMTQuNTk3IEUuMDEwNQpHMSBYMTY2LjE3NyBZMTE0LjY3MyBFLjAwNTM2CkcxIFgxNjYuNjU3IFkxMTQuNzI2IEUuMDEyNTgKRzEgWDE2Ny4yNDQgWTExNC42NSBFLjAxNTQyCkcxIFgxNjcuNzQ3IFkxMTQuNDI2IEUuMDE0MzUKO1dJRFRIOjAuNDgyMzE5CkcxIFgxNjcuOTQ0IFkxMTQuMjcyIEUuMDA3MDIKO1dJRFRIOjAuNTE0NjM5CkcxIFgxNjguMTQxIFkxMTQuMTE3IEUuMDA3NTQKRzEgWDE2OC40MTEgWTExMy42MiBFLjAxNzAyCjtXSURUSDowLjQ4Nzc3OApHMSBYMTY4LjY0NCBZMTEzLjE3NiBFLjAxNDI1CjtXSURUSDowLjQ3NjI0MQpHMSBYMTY4Ljg5NiBZMTEyLjYzOSBFLjAxNjQzCjtXSURUSDowLjQ5MTgxMgpHMSBYMTY5LjIwOSBZMTEyLjA1NyBFLjAxODk0CjtXSURUSDowLjUwNTU5OQpHMSBYMTY5LjQyNiBZMTExLjU5MyBFLjAxNTEyCkcxIFgxNjkuNzQ4IFkxMTEuMDc5IEUuMDE3OTEKO1dJRFRIOjAuNDkwOTYKRzEgWDE3MC4wNjEgWTExMC42NzMgRS4wMTQ2Nwo7V0lEVEg6MC40NjE2MzQKRzEgWDE3MC40MTIgWTExMC4yNDUgRS4wMTQ4Mgo7V0lEVEg6MC40MTQ0NzQKRzEgWDE3MC43MTMgWTEwOS44MzMgRS4wMTIxNgo7V0lEVEg6MC40MDQ1MTEKRzEgWDE3MC45OSBZMTA5LjQxNCBFLjAxMTY2CjtXSURUSDowLjQ0OTk5OQpHMSBYMTcxLjIyOCBZMTA4Ljk1OSBFLjAxMzM4CkcxIFgxNzEuMzA0IFkxMDguNTQ2IEUuMDEwOTQKRzEgWDE3MS4yNzIgWTEwOCBFLlkxNzAuMTUgRS4wMDg3MQpHMSBYOTMuNjc1IFkxNzAuMzE0IEUuMDA2ODYKRzEgWDkzLjMwNSBZMTcwLjQ0NCBFLjAxMDIyCkcxIFg5My4wNzQgWTE3MC40ODQgRS4wMDYxMQpHMSBYOTIuNjcgWTE3MC44ODIgRS4wMTQ3OApHMSBYOTIuMzYyIFkxNzEuMDA1IEUuMDA4NjQKRzEgWDkyLjIxNSBZMTcxLjI0NCBFLjAwNzMxCkcxIFg5MS43NDQgWTE3MS43MzEgRS4wMTc2NQpHMSBYOTEuNDM5IFkxNzEuOTE5IEUuMDA5MzQKRzEgWDkxLjAxMiBZMTcyLjIzMyBFLjAxMzgxCkcxIFg5MC40MjUgWTE3Mi4zODggRS4wMTU4MgpHMSBYODkuNzA4IFkxNzIuNDcgRS4wMTg4CkcxIFg4OS40NTQgWTE3Mi4zODEgRS4wMDcwMQpHMSBYODkuMjE0IFkxNzIuNjQzIEUuMDA5MjYKRzEgWDg4Ljg5NSBZMTcyLjczNCBFLjAwODY0CkcxIFg4OC41MiBZMTcyLjc0NCBFLjAwOTc3CkcxIFg4OC4yNyBZMTcyLjYyIEUuMDA3MjcKRzEgWDg4LjA4MyBZMTcyLjM2NCBFLjAwODI2CkcxIFg4OC4wNTUgWTE3Mi4yNTggRS4wMDI4NgpHMSBYODcuODc0IFkxNzIuMjE2IEUuMDA0ODQKRzEgWDg3LjY3MiBZMTcyLjA1NSBFLjAwNjczCkcxIFg4Ny40NzggWTE3Mi4yMjQgRS4wMDY3CkcxIFg4Ny4yNDMgWTE3Mi4yODMgRS4wMDYzMQpHMSBYODYuMjg2IFkxNzIuMjkxIEUuMDI0OTQKRzEgWDg2LjAxNSBZMTcyLjU1NCBFLjAwOTg0CkcxIFg4NS42NDggWTE3Mi42NjEgRS4wMDk5NgpHMSBYODUuMzQxIFkxNzIuNjEzIEUuMDA4MQpHMSBYODQuOTA0IFkxNzIuMzU5IEUuMDEzMTcKRzEgWDg0LjU3OCBZMTcyLjA3NSBFLjAxMTI3CkcxIFg4NC40NDEgWTE3MS43NTggRS4wMDkKRzEgWDg0LjQyMyBZMTcxLjY0NCBFLjAwMzAxCkcxIFg4NC4yMzQgWTE3MS41NzUgRS4wMDUyNApHMSBYODMuOTM4IFkxNzEuNjA1IEUuMDA3NzUKRzEgWDgzLjA4NSBZMTcxLjQ1OSBFLjAyMjU1CkcxIFg4Mi42MDggWTE3MS4zIEUuMDEzMQpHMSBYODIuMTYxIFkxNzEuMDcyIEUuMDEzMDcKRzEgWDgwLjc4NCBZMTcwLjQzNCBFLjAzOTU0CkcxIFg4MC4zNSBZMTcwLjMyMSBFLjAxMTY5Ck03MyBRNTMgUzIyNQpHMSBYNzkuODM2IFkxNzAuMTAxIEUuMDE0NTcKRzEgWDc5LjM2OCBZMTY5Ljg3MiBFLjAxMzU4CkcxIFg3OC4wNjkgWTE2OS4zODkgRS4wMzYxMQpHMSBYNzcuNTg3IFkxNjkuMjUyIEUuMDEzMDYKRzEgWDc3LjA4NyBZMTY5IEUuMDE0NTkKRzEgWDc2LjU5OCBZMTY4LjgyOCBFLjAxMzUxCkcxIFg3Ni4xMDMgWTE2OC41NyBFLjAxNDU0CkcxIFg3NS42NzYgWTE2OC40MjQgRS4wMTE3NgpHMSBYNzUuMTk3IFkxNjguMzQzIEUuMDEyNjYKRzEgWDc0LjU3MiBZMTY4LjA3NiBFLjAxNzcxCkcxIFg3My40ODMgWTE2Ny43MjQgRS4wMjk4MgpHMSBYNzMuMDEzIFkxNjcuNTQgRS4wMTMxNQpHMSBYNzIuMyBZMTY3LjM4MyBFLjAxOTAyCkcxIFg3MS43NDcgWTE2Ny4yMiBFLjAxNTAyCkcxIFg3MS4wMDIgWTE2Ny4wNiBFLjAxOTg1CkcxIFg3MC4xOTcgWTE2Ni45MjIgRS4wMjEyOApHMSBYNjkuNTU1IFkxNjYuODQ3IEUuMDE2ODQKRzEgWDY4Ljk3OCBZMTY2LjgyOCBZMTI1LjYwMyBFLjAxNzY3CkcxIFgxODcuNTY1IFkxMjUuODE3IEUuMDA4ODMKRzEgWDE4OC4xNjQgWTEyNS45MyBFLjAxNTg4CkcxIFgxODguMzk1IFkxMjYuMDI4IEUuMDA2NTQKRzEgWDE4OC41NDMgWTEyNi4wMDggRS4wMDM4OQpHMSBYMTg4Ljg1NiBZMTI1LjkxMyBFLjAwODUyCkcxIFgxODguODE1IFkxMjUuNTQ5IEUuMDA5NTQKRzEgWDE4OS4zOTEgWTEyNS40NDEgRS4wMTUyNwpHMSBYMTg5Ljc1OSBZMTI1LjQxNiBFLjAwOTYxCkcxIFgxODkuNzggWTEyNS4zOTYgRS4wMDA3NgpHMSBYMTkwLjA0MiBZMTI1LjM5NCBFLjAwNjgzCkcxIFgxOTAuMTM3IFkxMjUuMzYgRS4wMDI2MwpHMSBYMTkwLjQ1IFkxMjUuMDQzIEUuMDExNjEKRzEgWDE5MC44OTggWTEyNC4zMDcgRS4wMjI0NQpHMSBYMTkxLjM0NyBZMTIzLjQxOCBFLjAyNTk1CkcxIFgxOTEuNzk1IFkxMjMuMjE2IEUuMDEyOApHMSBYMTkyLjI0MyBZMTIzLjMwMSBFLjAxMTg4CkcxIFgxOTMuMTQgWTEyMy42NTggRS4wMjUxNgpHMSBYMTkzLjU4OCBZMTIzLjk1MyBFLjAxMzk4CkcxIFgxOTQuMDM2IFkxMjQuNDA2IEUuMDE2NgpHMSBYMTk0LjM0NCBZMTI0LjkxMSBFLjAxNTQxCkcxIFgxOTQuNzM4IFkxMjQuODUgRS4wMTAzOQpHMSBYMTk1Ljg2NCBZMTI0Ljc3NyBFLjAyOTQKRzEgWDE5Ni4zNyBZMTI0LjcxOSBFLjAxMzI3CkcxIFgxOTYuNzc1IFkxMjQuNzE5IEUuMDEwNTUKRzEgWDE5Ni44MTkgWTEyNC42ODUgRS4wMDE0NQo7V0lQRV9TVEFSVApHMSBGODY0MApHMSBYMTk2Ljc3NSBZMTI0LjcxOSBFLS4wMTI4NApHMSBYMTk2LjM3IFkxMjQuNzE5IEUtLjA5MzUyCkcxIFgxOTUuODY0IFkxMjQuNzc3IEUtLjExNzYKRzEgWDE5NC43MzggWTEyNC44NSBFLS4yNjA1NApHMSBYMTk0LjM0NCBZMTI0LjkxMSBFLS4wOTIwNgpHMSBYMTk0LjAzNiBZMTI0LjQwNiBFLS4xMzY1OApHMSBYMTkzLjc3MSBZMTI0LjEzOSBFLS4wODY4Ngo7V0lQRV9FTkQKRzEgWjEuMzUgRjcyMApHMSBYMTk3Ljc4MyBZMTIyLjc5NyBGMTA4MDAKRzEgWi45NSBGNzIwCkcxIEUuOCBGMjEwMApHMSBGNDgwMApHMSBYMTk4LjAwOCBZMTIyLjgzMyBFLjAwNTk0CkcxIFgxOTguMzc1IFkxMjMuMDEgRS4wMTA2MgpHMSBYMTk4LjY4NyBZMTIzLjA3OCBFLjAwODMyCkcxIFgxOTguODEyIFkxMjMuMTM2IEUuMDAzNTkKRzEgWDE5OC45NCBZMTIzLjIyOSBFLjAwNDEyCkcxIFgxOTguOTY4IFkxMjMuMjE2IEUuMDAwOApHMSBYMTk5LjQxNiBZMTIzLjMwMSBFLjAxMTg4CkcxIFgyMDAuMzEyIFkxMjMuNjU4IEUuMDI1MTMKRzEgWDIwMC43NjEgWTEyMy45NTMgRS4wMTQKRzEgWDIwMS4yMDkgWTEyNC40MDYgRS4wMTY2CkcxIFgyMDEuNjU3IFkxMjUuMTQzIEUuMDIyNDcKRzEgWDIwMi4xMDYgWTEyNi4wMzEgRS4wMjU5MwpHMSBYMjAyLjIxNyBZMTI2LjA4MSBFLjAwMzE3CkcxIFgyMDIuMjU3IFkxMjYuMjYyIEUuMDA0ODMKRzEgWDIwMi4yNzEgWTEyNi44MDMgRS4wMTQxCkcxIFgyMDIuMjY3IFkxMjYuOTgyIEUuMDA0NjcKRzEgWDIwMi4yMjEgWTEyNy4yMzQgRS5yb3AucmVtb3ZlKCk7JGJhc2VCYWNrZHJvcD0kYmFzZU1vZGFsPW51bGw7fQpyZXR1cm4gYmFzZUluZGV4W3R5cGVdKyh6SW5kZXhGYWN0b3IqcG9zKTt9fSgpKTtmdW5jdGlvbiB0YXJnZXRJc1NlbGYoY2FsbGJhY2spe3JldHVybiBmdW5jdGlvbihlKXtpZihlJiZ0aGlzPT09ZS50YXJnZXQpe3JldHVybiBjYWxsYmFjay5hcHBseSh0aGlzLGFyZ3VtZW50cyk7fX19CiQuZm4ubW9kYWxtYW5hZ2VyPWZ1bmN0aW9uKG9wdGlvbixhcmdzKXtyZXR1cm4gdGhpcy5lYWNoKGZ1bmN0aW9uKCl7dmFyICR0aGlzPSQodGhpcyksZGF0YT0kdGhpcy5kYXRhKCdtb2RhbG1hbmFnZXInKTtpZighZGF0YSkkdGhpcy5kYXRhKCdtb2RhbG1hbmFnZXInLChkYXRhPW5ldyBNb2RhbE1hbmFnZXIodGhpcyxvcHRpb24pKSk7aWYodHlwZW9mIG9wdGlvbj09PSdzdHJpbmcnKWRhdGFbb3B0aW9uXS5hcHBseShkYXRhLFtdLmNvbmNhdChhcmdzKSl9KX07JC5mbi5tb2RhbG1hbmFnZXIuZGVmYXVsdHM9e2JhY2tkcm9wTGltaXQ6OTk5LHJlc2l6ZTp0cnVlLHNwaW5uZXI6JzxkaXYgY2xhc3M9ImxvYWRpbmctc3Bpbm5lciBmYWRlIiBzdHlsZT0id2lkdGg6IDIwMHB4OyBtYXJnaW4tbGVmdDogLTEwMHB4OyI+PGRpdiBjbGFzcz0icHJvZ3Jlc3MgcHJvZ3Jlc3Mtc3RyaXBlZCBhY3RpdmUiPjxkaXYgY2xhc3M9ImJhciIgc3R5bGU9IndpZHRoOiAxMDAlOyI+PC9kaXY+PC9kaXY+PC9kaXY+JyxiYWNrZHJvcFRlbXBsYXRlOic8ZGl2IGNsYXNzPSJtb2RhbC1iYWNrZHJvcCIgLz4nfTskLmZuLm1vZGFsbWFuYWdlci5Db25zdHJ1Y3Rvcj1Nb2RhbE1hbmFnZXIKJChmdW5jdGlvbigpeyQoZG9jdW1lbnQpLm9mZignc2hvdy5icy5tb2RhbCcpLm9mZignaGlkZGVuLmJzLm1vZGFsJyk7fSk7fShqUXVlcnkpOwo7CgovLyBzb3VyY2U6IGpzL2xpYi9ib290c3RyYXAvYm9vdHN0cmFwLW1vZGFsLmpzCiFmdW5jdGlvbigkKXsidXNlIHN0cmljdCI7dmFyIE1vZGFsPWZ1bmN0aW9uKGVsZW1lbnQsb3B0aW9ucyl7dGhpcy5pbml0KGVsZW1lbnQsb3B0aW9ucyk7fTtNb2RhbC5wcm90b3R5cGU9e2NvbnN0cnVjdG9yOk1vZGFsLGluaXQ6ZnVuY3Rpb24oZWxlbWVudCxvcHRpb25zKXt2YXIgdGhhdD10aGlzO3RoaXMub3B0aW9ucz1vcHRpb25zO3RoaXMuJGVsZW1lbnQ9JChlbGVtZW50KS5kZWxlZ2F0ZSgnW2RhdGEtZGlzbWlzcz0ibW9kYWwiXScsJ2NsaWNrLmRpc21pc3MubW9kYWwnLCQucHJveHkodGhpcy5oaWRlLHRoaXMpKTt0aGlzLm9wdGlvbnMucmVtb3RlJiZ0aGlzLiRlbGVtZW50LmZpbmQoJy5tb2RhbC1ib2R5JykubG9hZCh0aGlzLm9wdGlvbnMucmVtb3RlLGZ1bmN0aW9uKCl7dmFyIGU9JC5FdmVudCgnbG9hZGVkJyk7dGhhdC4kZWxlbWVudC50cmlnZ2VyKGUpO30pO3ZhciBtYW5hZ2VyPXR5cGVvZiB0aGlzLm9wdGlvbnMubWFuYWdlcj09PSdmdW5jdGlvbic/dGhpcy5vcHRpb25zLm1hbmFnZXIuY2FsbCh0aGlzKTp0aGlzLm9wdGlvbnMubWFuYWdlcjttYW5hZ2VyPW1hbmFnZXIuYXBwZW5kTW9kYWw/bWFuYWdlcjokKApHMSBYMTI2LjM3IFkxMTEuMTQzIEUuMDE2MzEKRzEgWDEyMS45OCBZMTA2Ljc1MiBFLjE2MTc4CkcxIFgxMjIuMjQxIFkxMDYuNDIzIEUuMDEwOTQKRzEgWDEyNi45OTYgWTExMS4xNzggRS4xNzUyMQpHMSBYMTI3LjYyMiBZMTExLjIxMyBFLjAxNjM0CkcxIFgxMjIuNDU1IFkxMDYuMDQ2IEUuMTkwNApHMSBYMTIyLjYxIFkxMDUuNjEgRS4wMTIwNgpHMSBYMTI4LjI0NyBZMTExLjI0NyBFLjIwNzcxCkcxIFgxMjguODczIFkxMTEuMjgyIEUuMDE2MzQKRzEgWDEyMi42NzEgWTEwNS4wODEgRS4yMjg1MgpHMSBYMTIyLjQxIFkxMDQuMzY5IEUuMDE5NzYKO1dJRFRIOjAuMzcyMTkKRzEgWDEyMi4zOTggWTEwNC4yMTYgRS4wMDMyNQpHMSBYMTIyLjQyNSBZMTA0LjEyMSBFLjAwMjA5CjtXSURUSDowLjQxMTA5NQpHMSBYMTIyLjUxMSBZMTA0LjI2OCBFLjAwNDAyCjtXSURUSDowLjQ0OTk5OQpHMSBYMTIyLjU5OCBZMTA0LjQxNiBFLjAwNDQ3CkcxIFgxMjkuNDk4IFkxMTEuMzE3IEUuMjU0MjcKRzEgWDEzMC4xMjQgWTExMS4zNTIgRS4wMTYzNApHMSBYMTE4LjU0MiBZOTkuNzY5IEUuNDI2OApHMSBYMTE4LjM5MyBZOTkuMjA4IEUuMDE1MTIKO1dJRFRIOjAuMzcyMTkKRzEgWDExOC4zNzQgWTk5LjI3OSBFLjAwMTU2CkcxIFgxMTguNDQ1IFk5OS4yNiBFLjAwMTU2CkcxIFgxMTguNDM0IFk5OS4yMTkgRS4wMDA5CjtXSURUSDowLjQxMTA5NQpHMSBYMTE4LjU0MiBZOTkuMjUyIEUuMDAyNjcKO1dJRFRIOjAuNDQ5OTk5CkcxIFgxMTguNjQ5IFk5OS4yODYgRS4wMDI5MwpHMSBYMTMwLjc1IFkxMTEuMzg2IEUuNDQ1ODgKRzEgWDEzMS4zNzUgWTExMS40MjEgRS4wMTYzMQpHMSBYMTE5LjIwOSBZOTkuMjU1IEUuNDQ4MwpHMSBYMTE5Ljc2OSBZOTkuMjI0IEUuMDE0NjEKRzEgWDEzMi4wMDEgWTExMS40NTYgRS40NTA3MwpHMSBYMTMyLjYyNyBZMTExLjQ5MSBFLjAxNjM0CkcxIFgxMjAuMzI5IFk5OS4xOTMgRS40NTMxNgpHMSBYMTIwLjg4OCBZOTkuMTYxIEUuMDE0NTkKRzEgWDEzMy4yNTIgWTExMS41MjUgRS40NTU1OQpHMSBYMTMzLjg3OCBZMTExLjU2IEUuMDE2MzQKRzEgWDEyMS40NDggWTk5LjEzIEUuNDU4MDMKRzEgWDEyMi4wMDggWTk5LjA5OSBFLjAxNDYxCkcxIFgxMzQuNTAzIFkxMTEuNTk1IEUuNDYwNDQKRzEgWDEzNS40MzEgWTExMS42NDcgRjEwODAwCjtXSURUSDowLjM3MjE5CkcxIEY0ODAwCkcxIFgxMzUuNDQyIFkxMTEuNjg4IEUuMDAwOQpHMSBYMTM1LjM3MSBZMTExLjcwNyBFLjAwMTU2CkcxIFgxMzUuMzkgWTExMS42MzYgRS4wMDE1Ngo7V0lEVEg6MC40MTEwOTUKRzEgWDEzNS4yNiBZMTExLjYzMyBFLjAwMzA3CjtXSURUSDowLjQ0OTk5OQpHMSBYMTM1LjEyOSBZMTExLjYzIEUuMDAzNDEKRzEgWDEyMi41NjggWTk5LjA2OCBFLjQ2Mjg3CkcxIFgxMjMuMTI4IFk5OS4wMzcgRS4wMTQ2MQpHMSBYMTM1LjI3OCBZMTExLjE4OCBFLjQ0NzczCkcxIFgxMzUuMjc4IFkxMTAuNTk3IEUuMDE1NApHMSBYMTIzLjY4NyBZOTkuMDA2IEUuNDI3MTEKRzEgWDEyNC4yNDcgWTk4Ljk3NSBFLjAxLjcwNiBFLjAyNTUKRzEgWDE0Ny43NTggWTExMi43MzIgRS4wNDIwNgpHMSBYMTQ3Ljc1OCBZMTExLjYzNCBFLjAyODYxCkcxIFgxNDcuOTI0IFkxMTEuNDE0IEUuMDA3MTgKRzEgWDE0OC45OTIgWTExMC40MDIgRS4wMzgzNApHMSBYMTQ5Ljg0IFkxMDkuNTM1IEUuMDMxNgpHMSBYMTUwLjg1MSBZMTA4LjQwMyBFLjAzOTU1CkcxIFgxNTEuNzMxIFkxMDcuMzExIEUuMDM2NTQKO1dJRFRIOjAuNDkyNzE0CkcxIFgxNTEuODI0IFkxMDcuMjIyIEUuMDAzNwo7V0lEVEg6MC41MzU0MjgKRzEgWDE1MS45MTcgWTEwNy4xMzMgRS4wMDQwNAo7V0lEVEg6MC41NzgxNDIKRzEgWDE1Mi4wMSBZMTA3LjA0NCBFLjAwNDM4CjtXSURUSDowLjYyMDg1NgpHMSBYMTUyLjEwNCBZMTA2Ljk1NSBFLjAwNDc1CkcxIFgxNTIuNDM1IFkxMDYuNDk3IEUuMDIwNzQKO1dJRFRIOjAuNjAzNzM5CkcxIFgxNTMuMDMxIFkxMDUuNjIyIEUuMDM3NzQKO1dJRFRIOjAuNjQxNTU0CkcxIFgxNTMuMTggWTEwNS4yMjQgRS4wMTYxNQo7V0lEVEg6MC42NTc2MTcKRzEgWDE1My4xODcgWTEwNS4xNTEgRS4wMDI4NgpNMjA0IFMxMDAwCkcxIFgxNTIuNjg3IFkxMDUuMDM3IEYxMDgwMApNMjA0IFM4MDAKO1RZUEU6RXh0ZXJuYWwgcGVyaW1ldGVyCjtXSURUSDowLjQ0OTk5OQpHMSBGMTUwMAoKRzEgWDE1Mi41ODcgWTEwNC41OTggRS4wMTE3MwpHMSBYMTUyLjMwMiBZMTA0LjE3MyBFLjAxMzMzCkcxIFgxNTEuNDQyIFkxMDIuOTk0IEUuMDM4MDIKRzEgWDE1MC42NzkgWTEwMi4wNDIgRS4wMzE3OQpHMSBYMTUwLjA0MSBZMTAxLjMwMiBFLjAyNTQ2CkcxIFgxNDkuNTM4IFkxMDAuNzUzIEUuMDE5NApHMSBYMTQ4LjY4OCBZOTkuODg1IEUuMDMxNjUKRzEgWDE0Ny42NzMgWTk4LjkyNCBFLjAzNjQyCkcxIFgxNDcuMzM5IFk5OC40ODggRS4wMTQzMQpHMSBYMTQ3LjMzOSBZOTYuNDQ3IEUuMDUzMTgKRzEgWDE0Ny42NDQgWTk2LjY0MiBFLjAwOTQzCkcxIFgxNDguMzkyIFk5Ny4yMzMgRS4wMjQ4NApHMSBYMTQ5LjMwMyBZOTggRS4wMzEwMwpHMSBYMTUwLjE4NSBZOTguNzk2IEUuMDMwOTYKRzEgWDE1MC44OTkgWTk5LjQ4MiBFLjAyNTgKRzEgWDE1MS42OTggWTEwMC4yOTYgRS4wMjk3MgpHMSBYMTUyLjMzMSBZMTAwLjk5MyBFLjAyNDUzCkcxIEYxMjAwCkcxIFgxNTIuNTM5IFkxMDEuMzA3IEUuMDA5ODEKRzEgRjEwMjQuMDUzCkcxIFgxNTIuNzY5IFkxMDEuNjUzIEUuMDEwODMKRzEgWDE1Mi44MTQgWTEwMS44NDkgRS4wMDUyNApHMSBGMTIwMApHMSBYMTUyLjg5OCBZMTAyLjIxMiBFLjAwOTcxCkcxIEYxNTAwCkcxIFgxNTMuMDY1IFkxMDIuNTMxIEUuMDA5MzgKRzEgWDE1NC4wMTMgWTEwMy45MzUgRS4wNDQxNApHMSBYMTU0LjI2NiBZMTA0LjQyIEUuMDE0MjUKRzEgWDE1NC4zNTMgWTEwNC45NCBFLjAxMzc0CkcxIFgxNTQuMzQ0IFkxMDUuMjY1IEUuMDA4NDcKRzEgWDE1NC4yMSBZMTA1LjY5IEUuMDExNjEKRzEgWDE1My45MzggWTEwNi4xODEgRS4wMTQ2MwpHMSBYMTUzLjA2NCBZMTA3LjQ2OSBFLjA0MDU2CkcxIEYxNDY5LjY0OApHMSAwCkcxIEYyNDAwCkcxIFgxMzUuMjYyIFk5OS40MzkgRS4wMTk5NAo7V0lQRV9TVEFSVApHMSBGODY0MApHMSBYMTM1Ljg2OSBZOTguODMzIEUtLjE5ODA1CjtXSVBFX0VORApHMSBFLS42MDE5NSBGMjEwMApHMSBaMjUuMDUgRjcyMApHMSBYMTM1LjM5NyBZOTcuOCBGMTA4MDAKRzEgWjI0LjY1IEY3MjAKRzEgRS44IEYyMTAwCjtUWVBFOkludGVybmFsIGluZmlsbAo7V0lEVEg6MC40NQpHMSBGNDgwMApHMSBYMTM1LjQyMiBZOTcuNzMyIEUuMDAxODkKRzEgWDEzNS40MjIgWTk3LjU4NiBFLjAwMzgKRzEgWDEzNS4yNjIgWTk3LjU5NSBFLjAwNDE4CkcxIFgxMzUuMTAxIFk5Ny42MzIgRS4wMDQzCkcxIFgxMzUuMjY0IFk5Ny44MjcgRS4wMDY2MgpHMSBYMTM1LjMyNCBZOTguMDA0IEUuMDA0ODcKRzEgWDEzNC44MzcgWTk3LjY5MyBFLjAxNTA2CkcxIFgxMzQuODk4IFk5Ny42NzkgRS4wMDE2Mwo7V0lQRV9TVEFSVApHMSBGODY0MApHMSBYMTM0LjgzNyBZOTcuNjkzIEUtLjAxNDQ1CkcxIFgxMzUuMzI0IFk5OC4wMDQgRS0uMTMzNDIKRzEgWDEzNS4yNjQgWTk3LjgyNyBFLS4wNDMxNQpHMSBYMTM1LjEwMSBZOTcuNjMyIEUtLjA1ODY4CkcxIFgxMzUuMjYyIFk5Ny41OTUgRS0uMDM4MTQKRzEgWDEzNS40MjIgWTk3LjU4NiBFLS4wMzcKRzEgWDEzNS40MjIgWTk3LjczMiBFLS4wMzM3MQpHMSBYMTM1LjM5NyBZOTcuOCBFLS4wMTY3Mwo7V0lQRV9FTkQKRzEgRS0uNDI0NzIgRjIxMDAKRzEgWjI1LjA1IEY3MjAKRzEgWDExOC40ODMgWTk4Ljg3NyBGMTA4MDAKRzEgWjI0LjY1IEY3MjAKRzEgRS44IEYyMTAwCk0yMDQgUzgwMAo7VFlQRTpQZXJpbWV0ZXIKO1dJRFRIOjAuNTY2OTMxCkcxIEYxNTAwCkcxIFgxMTguNDMyIFk5OC45MiBFLjAwMjIyCjtXSURUSDowLjUyNzk1NApHMSBYMTE4LjM1OCBZOTguOTg0IEUuMDAzMDIKO1dJRFRIOjAuNDg4OTc3CkcxIFgxMTguMjg1IFk5OS4wNDggRS4wMDI3Nwo7V0lEVEg6MC40OTk0OQpHMSBYMTE4LjIwMiBZOTkuMTkxIEUuMDA0ODIKO1dJRFRIOjAuNTQ4OTgxCkcxIFgxMTguMTIgWTk5LjMzNCBFLjAwNTMxCjtXSURUSDowLjU5ODQ3MgpHMSBYMTE4LjAzOCBZOTkuNDc3IEUuMDA1ODIKO1dJRFRIOjAuNTk4NjczCkcxIFgxMTguMDYzIFk5OS44NTkgRS4wMTM1Mgo7V0lEVEg6MC41NDkxMTUKRzEgWDExOC4wODggWTEwMC4wNzQgRS4wMDY5OAo7V0lEVEg6MC40OTk1NTcKRzEgWDExOC4xMTMgWTEwMC4yODggRS4wMDYyOAo7V0lEVEg6MC40NDk5OTkKRzEgWDExOC4xMTMgWTEwMC43NiBFLjAxMjMKRzEgWDExNy4wOTggWTEwMC43NDEgRS4wMjY0NQpHMSBYMTE3LjA5NyBZMTAwLjEzNCBFLjAxNTgyCkcxIFgxMTcuMjEyIFkxMDAuMDg4IEUuMDAzMjMKO1dJRFRIOjAuNDg5MDMyCkcxIFgxMTcuMjc1IFkxMDAuMDA3IEUuMDAyOTIKO1dJRFRIOjAuNTI4MDY0CkcxIFgxMTcuMzM4IFk5OS45MjYgRS4wMDMxNwo7V0lEVEg6MC41NjcwOTYKRzEgWDExNy40MDIgWTk5Ljg0NSBFLjAwMzQ0CjtXSURUSDowLjYwNjEyOApHMSBYMTE3LjQ2NSBZOTkuNzY0IEUuMDAzNnRhKSB9LCBjc3M6IHsgYWN0aXZlOiAkcm9vdC5kaXN0YW5jZSgpID09PSAkZGF0YSB9LCBhdHRyOiB7IGlkOiAnY29udHJvbC1kaXN0YW5jZScgKyAkcm9vdC5zdHJpcERpc3RhbmNlRGVjaW1hbCgkZGF0YSkgfSI+PC9idXR0b24+CiAgICAgICAgICAgIDwhLS0gL2tvIC0tPgogICAgICAgIDwvZGl2PgogICAgPC9kaXY+CgogICAgPCEtLSBGZWVkIHJhdGUgLS0+CiAgICA8ZGl2IGlkPSJjb250cm9sLWpvZy1mZWVkcmF0ZSIgY2xhc3M9ImpvZy1wYW5lbCI+CiAgICAgICAgPGxhYmVsPkZlZWQgcmF0ZSBtb2RpZmllcjogPGEgY2xhc3M9InRleHQtaW5mbyIgaHJlZj0iamF2YXNjcmlwdDp2b2lkKDApIiBkYXRhLWJpbmQ9InBvcG92ZXI6IHtwbGFjZW1lbnQ6ICd0b3AnLCB0cmlnZ2VyOiAnaG92ZXInLCB0aXRsZTogJ1BsZWFzZSBub3RlIScsIGNvbnRlbnQ6ICdUaGUgZmVlZCByYXRlIGNhbiBvbmx5IGJlIHNldCwgaXQgY2Fubm90IGJlIHJlYWQgYmFjayBmcm9tIHRoZSBmaXJtd2FyZSBkdWUgdG8gYSBsaW1pdGF0aW9uIG9mIHRoZSBjb21tdW5pY2F0aW9uIHByb3RvY29sLiBUaGVyZSBpcyBubyB3YXkgdG8gc2hvdyB0aGUgY3VycmVudCBzZXR0aW5nLid9Ij48aSBjbGFzcz0iZmFzIGZhLWluZm8tY2lyY2xlIj48L2k+PC9hPjwvbGFiZWw+CiAgICAgICAgPGRpdiBjbGFzcz0iaW5wdXQtYXBwZW5kIGNvbnRyb2wtYm94Ij4KICAgICAgICAgICAgPGlucHV0IHR5cGU9Im51bWJlciIgY2xhc3M9ImlucHV0LW1pbmkiIG1pbj0iMSIgc3RlcD0iMSIgZGF0YS1iaW5kPSJ0ZXh0SW5wdXQ6IGZlZWRSYXRlLCBldmVudDogeyBibHVyOiByZXNldEZlZWRSYXRlRGlzcGxheSwgZm9jdXM6IGNhbmNlbEZlZWRSYXRlRGlzcGxheVJlc2V0IH0sIGNzczogeyBwdWxzYXRlX3RleHRfb3BhY2l0eTogZmVlZFJhdGVSZXNldHRlcigpICE9PSB1bmRlZmluZWQgfSI+CiAgICAgICAgICAgIDxzcGFuIGNsYXNzPSJhZGQtb24iPiU8L3NwYW4+CiAgICAgICAgICAgIDxidXR0b24gY2xhc3M9ImJ0biIgZGF0YS1iaW5kPSJlbmFibGU6IGlzT3BlcmF0aW9uYWwoKSAmJiBmZWVkUmF0ZSgpLCBjbGljazogZnVuY3Rpb24oKSB7ICRyb290LnNlbmRGZWVkUmF0ZUNvbW1hbmQoKSB9Ij5TZXQ8L2J1dHRvbj4KICAgICAgICA8L2Rpdj4KICAgIDwvZGl2Pgo8L2Rpdj4KPCEtLSBFeHRydXNpb24gY29udHJvbCBwYW5lbCAtLT4KPGRpdiBpZD0iY29udHJvbC1qb2ctZXh0cnVzaW9uIiBjbGFzcz0iam9nLXBhbmVsIiBzdHlsZT0iZGlzcGxheTogbm9uZTsiIGRhdGEtYmluZD0idmlzaWJsZTogbG9naW5TdGF0ZS5oYXNQZXJtaXNzaW9uS28oYWNjZXNzLnBlcm1pc3Npb25zLkNPTlRST0wpKCkgJiYgdG9vbHMoKS5sZW5ndGggPiAwIj4KICAgIDxoMT5Ub28=" diff --git a/octoprint_octoeverywhere/__init__.py b/octoprint_octoeverywhere/__init__.py index e412f79..5ffc765 100644 --- a/octoprint_octoeverywhere/__init__.py +++ b/octoprint_octoeverywhere/__init__.py @@ -14,6 +14,7 @@ from octoeverywhere.octohttprequest import OctoHttpRequest from octoeverywhere.notificationshandler import NotificationsHandler from octoeverywhere.octopingpong import OctoPingPong +from octoeverywhere.compression import Compression from octoeverywhere.telemetry import Telemetry from octoeverywhere.sentry import Sentry from octoeverywhere.mdns import MDns @@ -170,6 +171,9 @@ def on_startup(self, host, port): # Set the printer id to Sentry. Sentry.SetPrinterId(printerId) + # Setup compression + Compression.Init(self._logger, self.get_plugin_data_folder()) + # Init the static local auth helper LocalAuth.Init(self._logger, self._user_manager) diff --git a/octoprint_octoeverywhere/__main__.py b/octoprint_octoeverywhere/__main__.py index 2d53275..1d1092f 100644 --- a/octoprint_octoeverywhere/__main__.py +++ b/octoprint_octoeverywhere/__main__.py @@ -9,6 +9,7 @@ from octoeverywhere.octohttprequest import OctoHttpRequest from octoeverywhere.commandhandler import CommandHandler from octoeverywhere.octopingpong import OctoPingPong +from octoeverywhere.compression import Compression from octoeverywhere.telemetry import Telemetry from octoeverywhere.sentry import Sentry from octoeverywhere.mdns import MDns @@ -36,10 +37,10 @@ # For local setups, use these vars to configure things. LocalServerAddress = None -#LocalServerAddress = "octoeverywhere.dev" +#LocalServerAddress = "192.168.1.3" OctoPrintIp = None -OctoPrintIp = "192.168.1.12" +OctoPrintIp = "192.168.1.10" OctoPrintPort = None OctoPrintPort = 80 @@ -169,6 +170,9 @@ def GeneratePrinterId(): if LocalServerAddress is not None: Telemetry.SetServerProtocolAndDomain("http://"+LocalServerAddress) + # Setup compression + Compression.Init(logger, PluginFilePathRoot) + # Init the mdns client MDns.Init(logger, PluginFilePathRoot) #MDns.Get().Test() @@ -181,7 +185,7 @@ def GeneratePrinterId(): signal.signal(signal.SIGINT, SignalHandler) # Dev props - OctoEverywhereWsUri = "ws://starport-v1.octoeverywhere.com/octoclientws" + OctoEverywhereWsUri = "wss://starport-v1.octoeverywhere.com/octoclientws" # Setup the http requester OctoHttpRequest.SetLocalHttpProxyPort(80) diff --git a/octoprint_octoeverywhere/octoprintwebcamhelper.py b/octoprint_octoeverywhere/octoprintwebcamhelper.py index f2bb558..9ed50c5 100644 --- a/octoprint_octoeverywhere/octoprintwebcamhelper.py +++ b/octoprint_octoeverywhere/octoprintwebcamhelper.py @@ -39,7 +39,7 @@ def GetWebcamConfig(self): self.Logger.info("OctoPrintWebcamHelper has no OctoPrintSettingsObject. Returning default address.") baseUrl = f"http://{OctoHttpRequest.GetLocalhostAddress()}" return [ - WebcamSettingItem(f"{baseUrl}/webcam/?action=snapshot", f"{baseUrl}/webcam/?action=stream", False, False, 0) + WebcamSettingItem("Dev", f"{baseUrl}/webcam/?action=snapshot", f"{baseUrl}/webcam/?action=stream") ] # Since OctoPrint 1.9.0+ needs to call plugins to return webcam settings, we want to reduce how often we make the call. diff --git a/octoprint_octoeverywhere/slipstream.py b/octoprint_octoeverywhere/slipstream.py index e3d9732..82584ac 100644 --- a/octoprint_octoeverywhere/slipstream.py +++ b/octoprint_octoeverywhere/slipstream.py @@ -1,6 +1,5 @@ import threading import time -import zlib from octoeverywhere.sentry import Sentry from octoeverywhere.compat import Compat @@ -9,6 +8,7 @@ from octoeverywhere.WebStream.octoheaderimpl import HeaderHelper from octoeverywhere.WebStream.octoheaderimpl import BaseProtocol from octoeverywhere.octostreammsgbuilder import OctoStreamMsgBuilder +from octoeverywhere.compression import Compression, CompressionContext from .localauth import LocalAuth @@ -140,6 +140,9 @@ def GetCachedOctoHttpResult(self, httpInitialContext): # We have our path, check if it's in the map with self.Lock: if path in self.Cache: + # Note that this object can be updated! + # There's only once case right now, there's logic that will compare the cache header and convert the + # Object into a 304 response, which will strip some headers and the body buffer. self.Logger.debug("Slipstream returning cached content for "+path) return self.Cache[path] @@ -207,16 +210,19 @@ def _GetAndProcessIndex(self): with self.Lock: self.Cache[Slipstream.IndexCachePath] = indexResult + # It's no ideal that we need to de-compress this, but it's fine since we are in the background. + indexBodyStr = None + with CompressionContext(self.Logger) as compressionContext: + # For decompression, we give the pre-compressed size and the compression type. The True indicates this it the only message, so it's all here. + indexBodyBytes = Compression.Get().Decompress(compressionContext, indexBodyBuffer, indexResult.BodyBufferPreCompressSize, True, indexResult.BodyBufferCompressionType) + indexBodyStr = indexBodyBytes.decode(encoding="utf-8") + # Set the result to None to make sure we don't use it anymore. indexResult = None # Now process the index to see if there's more we should cache. # We explicitly look for known files in the index should reference that are large. # If we don't find them, no big deal. - # It's no ideal that we need to de-compress this, but it's fine since we are in the background. - # PY2 zlib.decompress can't accept a bytearray, so we must convert them before compressing. - # This isn't ideal, but not a big deal since this is in the background. - indexBodyStr = zlib.decompress(indexBodyBuffer).decode(errors="ignore") for subPath in Slipstream.OptionalPartialCachePaths: # This function will try to find the full url or path in the index body, including the query string. fullPath = self.TryToFindFullUrl(indexBodyStr, subPath) @@ -239,7 +245,7 @@ def _GetAndProcessIndex(self): # On success returns the fully ready OctoHttpResult object. # On failure, returns None - def _GetCacheReadyOctoHttpResult(self, url): + def _GetCacheReadyOctoHttpResult(self, url) -> OctoHttpRequest.Result: success = False try: # Take the starting time. @@ -302,13 +308,18 @@ def _GetCacheReadyOctoHttpResult(self, url): return None # Do the compression. - # See the compression chat in the main http stream class for tradeoffs about complexity. ogSize = len(buffer) compressStart = time.time() - buffer = zlib.compress(buffer, 7) + compressResult = None + with CompressionContext(self.Logger) as compressionContext: + # It's important to set the full compression size, because the compression system will use + # it to better optimize the compression and know that we will be sending the full data. + compressionContext.SetTotalCompressedSizeOfData(len(buffer)) + compressResult = Compression.Get().Compress(compressionContext, buffer) + buffer = compressResult.Bytes # Set the buffer into the response so the http request logic can use it. - octoHttpResult.SetFullBodyBuffer(buffer, True, ogSize) + octoHttpResult.SetFullBodyBuffer(buffer, compressResult.CompressionType, ogSize) requestDuration = compressStart - start compressDuration = time.time() - compressStart diff --git a/py_installer/Installer.py b/py_installer/Installer.py index c4ee1c7..c6f405b 100644 --- a/py_installer/Installer.py +++ b/py_installer/Installer.py @@ -14,6 +14,7 @@ from .Frontend import Frontend from .Uninstall import Uninstall from .Ffmpeg import Ffmpeg +from .ZStandard import ZStandard class Installer: @@ -113,6 +114,9 @@ def _RunInternal(self): uninstall.DoUninstall(context) return + # Since this runs an async thread, kick it off now so it can start working. + ZStandard.TryToInstallZStandardAsync(context) + # Next step is to discover and fill out the moonraker config file path and service file name. # If we are doing an companion or bambu setup, we need the user to help us input the details to the external moonraker IP or bambu printer. # This is the hardest part of the setup, because it's highly dependent on the system and different moonraker setups. @@ -154,6 +158,10 @@ def _RunInternal(self): # Just before we start (or restart) the service, ensure all of the permission are set correctly permissions.EnsureFinalPermissions(context) + # If there was an install running, wait for it to finish now, before the service starts. + # For most installs, the user will take longer to add the info than it takes to install zstandard. + ZStandard.WaitForInstallToComplete() + # We are fully configured, create the service file and it's dependent files. service = Service() service.Install(context) diff --git a/py_installer/Updater.py b/py_installer/Updater.py index 6cc9165..1626e3e 100644 --- a/py_installer/Updater.py +++ b/py_installer/Updater.py @@ -11,6 +11,7 @@ from .Service import Service from .Util import Util from .Ffmpeg import Ffmpeg +from .ZStandard import ZStandard # # This class is responsible for doing updates for all local, companions, and bambu connect plugins on this local system. @@ -30,6 +31,9 @@ class Updater: def DoUpdate(self, context:Context): Logger.Header("Starting Update Logic") + # Since this takes a while, kick it off now. The pip install can take upwards of 30 seconds. + ZStandard.TryToInstallZStandardAsync(context) + # Enumerate all service file to find any local plugins, Sonic Pad plugins, companion service files, and bambu service files, since all service files contain this name. # Note GetServiceFileFolderPath will return dynamically based on the OsType detected. # Use sorted, so the results are in a nice user presentable order. @@ -48,6 +52,10 @@ def DoUpdate(self, context:Context): # On any system, try to install or update ffmpeg. Ffmpeg.TryToInstallFfmpeg(context) + # Before we restart the plugins, wait for the zstandard install to be done. + # Give the updater extra time to work, since it's much shorter + ZStandard.WaitForInstallToComplete(timeoutSec==20.0) + Logger.Info("We found the following plugins to update:") for s in foundOeServices: Logger.Info(f" {s}") diff --git a/py_installer/ZStandard.py b/py_installer/ZStandard.py new file mode 100644 index 0000000..f7d4451 --- /dev/null +++ b/py_installer/ZStandard.py @@ -0,0 +1,78 @@ +import sys +import time +import subprocess +import threading +import multiprocessing + +from octoeverywhere.compression import Compression + +from .Util import Util +from .Logging import Logger +from .Context import Context, OsTypes + +# A helper class to make sure the optional zstandard lib and deps are installed. +class ZStandard: + + # If there's an installer thread, it will be stored here. + _InstallThread = None + + # Tries to install zstandard, but this won't fail if the install fails. + # The PIP install can take quite a long time (20-30 seconds) so we run in async. + @staticmethod + def TryToInstallZStandardAsync(context:Context) -> None: + # We don't even try installing on K1 or SonicPad, we know it fail. + if context.OsType == OsTypes.K1 or context.OsType == OsTypes.SonicPad: + return + + # We also don't try install on systems with 2 cores or less, since it's too much work and the OS most of the time + # Can't support zstandard because there's no pre-made binary, it can't be built, and the install process will take too long. + if multiprocessing.cpu_count() < Compression.ZStandardMinCoreCountForInstall: + return + + # Since the pip install can take a long time, do the install process async. + ZStandard._InstallThread = threading.Thread(target=ZStandard._InstallThread, daemon=True) + ZStandard._InstallThread.start() + + + @staticmethod + def WaitForInstallToComplete(timeoutSec:float=10.0) -> None: + # See if we started a thread. + t = ZStandard._InstallThread + if t is None: + return + + # If we did, report and try to join it. + # If this fails, it's no big deal, because the plugin runtime will also try to install zstandard. + Logger.Info("Finishing install... this might take a moment...") + try: + t.join(timeout=timeoutSec) + except Exception as e: + Logger.Debug(f"Failed to join ztd installer thread. {str(e)}") + + + @staticmethod + def _InstallThread() -> None: + try: + # Try to install the system package, if possible. This might bring in a binary. + # If this fails, the PY package might be able to still bring in a pre-built binary. + Logger.Debug("Installing zstandard, this might take a moment...") + startSec = time.time() + (returnCode, stdOut, stdError) = Util.RunShellCommand("sudo apt-get install zstd -y", False) + Logger.Debug(f"Zstandard apt install result. Code: {returnCode}, StdOut: {stdOut}, StdErr: {stdError}") + + # Now try to install the PY package. + # NOTE: Use the same logic as we do in the Compression class. + # Only allow blocking up to 20 seconds, so we don't hang the installer too long. + result = subprocess.run([sys.executable, '-m', 'pip', 'install', Compression.ZStandardPipPackageString], timeout=30.0, check=False, capture_output=True) + Logger.Debug(f"Zstandard PIP install result. Code: {result.returncode}, StdOut: {result.stdout}, StdErr: {result.stderr}") + + # Report the status to the installer log. + if result.returncode == 0: + Logger.Debug(f"zStandard successfully installed/updated. It took {str(round(time.time()-startSec, 2))} seconds.") + return + + # Tell the user, but this is a best effort, so if it fails we don't care. + # Any user who wants to use RTSP and doesn't have ffmpeg installed can use our help docs to install it. + Logger.Debug(f"We didn't install zstandard. It took {str(round(time.time()-startSec, 2))} seconds. Output: {result.stderr}") + except Exception as e: + Logger.Debug(f"Error installing zstandard. {str(e)}") diff --git a/requirements.txt b/requirements.txt index 3ce1612..1d03d2c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,7 @@ dnspython>=2.3.0 httpx>=0.24.1,<0.26.0 urllib3>=1.26.15,<2.0.0 sentry-sdk>=1.19.1,<2 +#zstandard>=0.22.0,<0.23.0 # The following are required only for Moonraker configparser # Only used for Bambu Connect diff --git a/setup.py b/setup.py index 55f33c1..19932e0 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this single version string is used by all of the plugins in OctoEverywhere! -plugin_version = "3.3.7" +plugin_version = "3.4.0" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module @@ -64,6 +64,10 @@ # urllib3 # There is a bug with parsing headers in versions older than 1.26.? (https://github.com/diyan/pywinrm/issues/269). At least 1.26.6 fixes it, ubt we decide to just stick with a newer version. # This must be less than 2.0.0, because 2.0.0 requires open ssl 1.1.1+, which the sonic pad doesn't have. +# zstandard +# zstandard gives us great compression that's super fast, but it requires a native lib to installed. The PY package will come with a lib and or try to build it, but we can also install it via apt-get. +# For the complexity, we can't list it as a required install, since it won't work on some platforms. So instead we will try to install it during runtime, and then it will be used after the following restart. +# The package version is defined in octoeverywhere.compression.ZStandardPipPackageString # # Other lib version notes: # pillow - We don't require a version of pillow because we don't want to mess with other plugins and we use basic, long lived APIs.\ @@ -83,7 +87,8 @@ "dnspython>=2.3.0", "httpx>=0.24.1,<0.26.0", "urllib3>=1.26.18,<2.0.0", - "sentry-sdk>=1.19.1,<2" + "sentry-sdk>=1.19.1,<2", + #"zstandard" - optional lib see notes ] ### -------------------------------------------------------------------------------------------------------------------- From 6ea4c688fb82b6900eda16ffbe9451223c09cb7b Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Wed, 19 Jun 2024 07:56:05 -0700 Subject: [PATCH 108/328] Minor bug fix. --- py_installer/Updater.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py_installer/Updater.py b/py_installer/Updater.py index 1626e3e..ddc1f5e 100644 --- a/py_installer/Updater.py +++ b/py_installer/Updater.py @@ -54,7 +54,7 @@ def DoUpdate(self, context:Context): # Before we restart the plugins, wait for the zstandard install to be done. # Give the updater extra time to work, since it's much shorter - ZStandard.WaitForInstallToComplete(timeoutSec==20.0) + ZStandard.WaitForInstallToComplete(timeoutSec=20.0) Logger.Info("We found the following plugins to update:") for s in foundOeServices: From 56a43ea7372c981ae07154becbfdea9c02261ccc Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Wed, 19 Jun 2024 08:11:33 -0700 Subject: [PATCH 109/328] Converging the install logic to a single thread to prevent packag lock issues. --- .vscode/settings.json | 1 + py_installer/Ffmpeg.py | 31 --------- py_installer/Installer.py | 11 +-- py_installer/OptionalDepsInstaller.py | 97 +++++++++++++++++++++++++++ py_installer/Updater.py | 10 +-- py_installer/ZStandard.py | 78 --------------------- 6 files changed, 104 insertions(+), 124 deletions(-) delete mode 100644 py_installer/Ffmpeg.py create mode 100644 py_installer/OptionalDepsInstaller.py delete mode 100644 py_installer/ZStandard.py diff --git a/.vscode/settings.json b/.vscode/settings.json index f5e48b6..9a968e6 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -41,6 +41,7 @@ "crypo", "Damerell", "decompressor", + "decompressors", "deps", "devel", "devs", diff --git a/py_installer/Ffmpeg.py b/py_installer/Ffmpeg.py deleted file mode 100644 index 606aaa6..0000000 --- a/py_installer/Ffmpeg.py +++ /dev/null @@ -1,31 +0,0 @@ -import time - -from .Util import Util -from .Logging import Logger -from .Context import Context, OsTypes - -# A helper class to make sure ffmpeg is installed. -class Ffmpeg: - - # Tries to install ffmpeg, but this won't fail if the install fails. - @staticmethod - def TryToInstallFfmpeg(context:Context): - - # We don't even try installing ffmpeg on K1 or SonicPad. - if context.OsType == OsTypes.K1 or context.OsType == OsTypes.SonicPad: - return - - # Try to install or upgrade. - Logger.Info("Installing ffmpeg, this might take a moment...") - startSec = time.time() - (returnCode, stdOut, stdError) = Util.RunShellCommand("sudo apt-get install ffmpeg -y", False) - - # Report the status to the installer log. - Logger.Debug(f"FFmpeg install result. Code: {returnCode}, StdOut: {stdOut}, StdErr: {stdError}") - if returnCode == 0: - Logger.Info(f"Ffmpeg successfully installed/updated. It took {str(round(time.time()-startSec, 2))} seconds.") - return - - # Tell the user, but this is a best effort, so if it fails we don't care. - # Any user who wants to use RTSP and doesn't have ffmpeg installed can use our help docs to install it. - Logger.Info(f"We didn't install ffmpeg. It took {str(round(time.time()-startSec, 2))} seconds. Output: {stdError}") diff --git a/py_installer/Installer.py b/py_installer/Installer.py index c6f405b..f5bdaad 100644 --- a/py_installer/Installer.py +++ b/py_installer/Installer.py @@ -13,8 +13,7 @@ from .TimeSync import TimeSync from .Frontend import Frontend from .Uninstall import Uninstall -from .Ffmpeg import Ffmpeg -from .ZStandard import ZStandard +from .OptionalDepsInstaller import OptionalDepsInstaller class Installer: @@ -115,7 +114,7 @@ def _RunInternal(self): return # Since this runs an async thread, kick it off now so it can start working. - ZStandard.TryToInstallZStandardAsync(context) + OptionalDepsInstaller.TryToInstallDepsAsync(context) # Next step is to discover and fill out the moonraker config file path and service file name. # If we are doing an companion or bambu setup, we need the user to help us input the details to the external moonraker IP or bambu printer. @@ -144,10 +143,6 @@ def _RunInternal(self): frontend = Frontend() frontend.DoFrontendSetup(context) - # We need ffmpeg for the Bambu Connect X1 streaming or any user who wants to use a RTSP camera. - # Installing ffmpeg is best effort and not required for the plugin to work. - Ffmpeg.TryToInstallFfmpeg(context) - # Before we start the service, check if the secrets config file already exists and if a printer id already exists. # This will indicate if this is a fresh install or not. context.ExistingPrinterId = Linker.GetPrinterIdFromServiceSecretsConfigFile(context) @@ -160,7 +155,7 @@ def _RunInternal(self): # If there was an install running, wait for it to finish now, before the service starts. # For most installs, the user will take longer to add the info than it takes to install zstandard. - ZStandard.WaitForInstallToComplete() + OptionalDepsInstaller.WaitForInstallToComplete() # We are fully configured, create the service file and it's dependent files. service = Service() diff --git a/py_installer/OptionalDepsInstaller.py b/py_installer/OptionalDepsInstaller.py new file mode 100644 index 0000000..516ab88 --- /dev/null +++ b/py_installer/OptionalDepsInstaller.py @@ -0,0 +1,97 @@ +import sys +import time +import subprocess +import threading +import multiprocessing + +from octoeverywhere.compression import Compression + +from .Util import Util +from .Logging import Logger +from .Context import Context, OsTypes + +# A helper class to make sure the optional dependencies are installed, like zstandard and ffmpeg. +# Note that ideally all apt-get and pip installs should be done here, to prevent package lock conflicts. +class OptionalDepsInstaller: + + # If there's an installer thread, it will be stored here. + _InstallThread = None + + # Tries to install zstandard and ffmpeg, but this won't fail if the install fails. + # The PIP install can take quite a long time (20-30 seconds) so we run in async. + @staticmethod + def TryToInstallDepsAsync(context:Context) -> None: + # Since the pip and apt install can take a long time, do the install process async. + OptionalDepsInstaller._InstallThread = threading.Thread(target=OptionalDepsInstaller._InstallThread, args=(context,), daemon=True) + OptionalDepsInstaller._InstallThread.start() + + + @staticmethod + def WaitForInstallToComplete(timeoutSec:float=10.0) -> None: + # See if we started a thread. + t = OptionalDepsInstaller._InstallThread + if t is None: + return + + # If we did, report and try to join it. + # If this fails, it's no big deal, because the plugin runtime will also try to install zstandard. + Logger.Info("Finishing install... this might take a moment...") + try: + t.join(timeout=timeoutSec) + except Exception as e: + Logger.Debug(f"Failed to join optional installer thread. {str(e)}") + + + @staticmethod + def _InstallThread(context:Context) -> None: + # Try to install ffmpeg, this is required for RTSP streaming. + OptionalDepsInstaller._DoFfmpegInstall(context) + + # Try to install zstandard, this is optional but recommended. + OptionalDepsInstaller._InstallZStandard(context) + + + @staticmethod + def _InstallZStandard(context:Context) -> None: + try: + # We don't even try installing on K1 or SonicPad, we know it fail. + if context.OsType == OsTypes.K1 or context.OsType == OsTypes.SonicPad: + return + + # We don't try install zstandard on systems with 2 cores or less, since it's too much work and the OS most of the time + # Can't support zstandard because there's no pre-made binary, it can't be built, and the install process will take too long. + if multiprocessing.cpu_count() < Compression.ZStandardMinCoreCountForInstall: + return + + # Try to install the system package, if possible. This might bring in a binary. + # If this fails, the PY package might be able to still bring in a pre-built binary. + Logger.Debug("Installing zstandard, this might take a moment...") + startSec = time.time() + (returnCode, stdOut, stdError) = Util.RunShellCommand("sudo apt-get install zstd -y", False) + Logger.Debug(f"Zstandard apt install result. Code: {returnCode}, StdOut: {stdOut}, StdErr: {stdError}") + + # Now try to install the PY package. + # NOTE: Use the same logic as we do in the Compression class. + # Only allow blocking up to 20 seconds, so we don't hang the installer too long. + result = subprocess.run([sys.executable, '-m', 'pip', 'install', Compression.ZStandardPipPackageString], timeout=30.0, check=False, capture_output=True) + Logger.Debug(f"Zstandard PIP install result. Code: {result.returncode}, StdOut: {result.stdout}, StdErr: {result.stderr}, Time: {time.time()-startSec}") + + except Exception as e: + Logger.Debug(f"Error installing zstandard. {str(e)}") + + + @staticmethod + def _DoFfmpegInstall(context:Context) -> None: + try: + # We don't even try installing on K1 or SonicPad, we know it fail. + if context.OsType == OsTypes.K1 or context.OsType == OsTypes.SonicPad: + return + + # Try to install ffmpeg, this is required for RTSP streaming. + Logger.Debug("Installing ffmpeg, this might take a moment...") + startSec = time.time() + (returnCode, stdOut, stdError) = Util.RunShellCommand("sudo apt-get install ffmpeg -y", False) + # Report the status to the installer log. + Logger.Debug(f"FFmpeg install result. Code: {returnCode}, StdOut: {stdOut}, StdErr: {stdError}, Time: {time.time()-startSec}") + except Exception as e: + Logger.Debug(f"Error installing ffmpeg. {str(e)}") diff --git a/py_installer/Updater.py b/py_installer/Updater.py index ddc1f5e..909cd64 100644 --- a/py_installer/Updater.py +++ b/py_installer/Updater.py @@ -10,8 +10,7 @@ from .Paths import Paths from .Service import Service from .Util import Util -from .Ffmpeg import Ffmpeg -from .ZStandard import ZStandard +from .OptionalDepsInstaller import OptionalDepsInstaller # # This class is responsible for doing updates for all local, companions, and bambu connect plugins on this local system. @@ -32,7 +31,7 @@ def DoUpdate(self, context:Context): Logger.Header("Starting Update Logic") # Since this takes a while, kick it off now. The pip install can take upwards of 30 seconds. - ZStandard.TryToInstallZStandardAsync(context) + OptionalDepsInstaller.TryToInstallDepsAsync(context) # Enumerate all service file to find any local plugins, Sonic Pad plugins, companion service files, and bambu service files, since all service files contain this name. # Note GetServiceFileFolderPath will return dynamically based on the OsType detected. @@ -49,12 +48,9 @@ def DoUpdate(self, context:Context): Logger.Warn("No local, companion, or Bambu Connect plugins were found on this device.") raise Exception("No local, companion, or Bambu Connect plugins were found on this device.") - # On any system, try to install or update ffmpeg. - Ffmpeg.TryToInstallFfmpeg(context) - # Before we restart the plugins, wait for the zstandard install to be done. # Give the updater extra time to work, since it's much shorter - ZStandard.WaitForInstallToComplete(timeoutSec=20.0) + OptionalDepsInstaller.WaitForInstallToComplete(timeoutSec=30.0) Logger.Info("We found the following plugins to update:") for s in foundOeServices: diff --git a/py_installer/ZStandard.py b/py_installer/ZStandard.py deleted file mode 100644 index f7d4451..0000000 --- a/py_installer/ZStandard.py +++ /dev/null @@ -1,78 +0,0 @@ -import sys -import time -import subprocess -import threading -import multiprocessing - -from octoeverywhere.compression import Compression - -from .Util import Util -from .Logging import Logger -from .Context import Context, OsTypes - -# A helper class to make sure the optional zstandard lib and deps are installed. -class ZStandard: - - # If there's an installer thread, it will be stored here. - _InstallThread = None - - # Tries to install zstandard, but this won't fail if the install fails. - # The PIP install can take quite a long time (20-30 seconds) so we run in async. - @staticmethod - def TryToInstallZStandardAsync(context:Context) -> None: - # We don't even try installing on K1 or SonicPad, we know it fail. - if context.OsType == OsTypes.K1 or context.OsType == OsTypes.SonicPad: - return - - # We also don't try install on systems with 2 cores or less, since it's too much work and the OS most of the time - # Can't support zstandard because there's no pre-made binary, it can't be built, and the install process will take too long. - if multiprocessing.cpu_count() < Compression.ZStandardMinCoreCountForInstall: - return - - # Since the pip install can take a long time, do the install process async. - ZStandard._InstallThread = threading.Thread(target=ZStandard._InstallThread, daemon=True) - ZStandard._InstallThread.start() - - - @staticmethod - def WaitForInstallToComplete(timeoutSec:float=10.0) -> None: - # See if we started a thread. - t = ZStandard._InstallThread - if t is None: - return - - # If we did, report and try to join it. - # If this fails, it's no big deal, because the plugin runtime will also try to install zstandard. - Logger.Info("Finishing install... this might take a moment...") - try: - t.join(timeout=timeoutSec) - except Exception as e: - Logger.Debug(f"Failed to join ztd installer thread. {str(e)}") - - - @staticmethod - def _InstallThread() -> None: - try: - # Try to install the system package, if possible. This might bring in a binary. - # If this fails, the PY package might be able to still bring in a pre-built binary. - Logger.Debug("Installing zstandard, this might take a moment...") - startSec = time.time() - (returnCode, stdOut, stdError) = Util.RunShellCommand("sudo apt-get install zstd -y", False) - Logger.Debug(f"Zstandard apt install result. Code: {returnCode}, StdOut: {stdOut}, StdErr: {stdError}") - - # Now try to install the PY package. - # NOTE: Use the same logic as we do in the Compression class. - # Only allow blocking up to 20 seconds, so we don't hang the installer too long. - result = subprocess.run([sys.executable, '-m', 'pip', 'install', Compression.ZStandardPipPackageString], timeout=30.0, check=False, capture_output=True) - Logger.Debug(f"Zstandard PIP install result. Code: {result.returncode}, StdOut: {result.stdout}, StdErr: {result.stderr}") - - # Report the status to the installer log. - if result.returncode == 0: - Logger.Debug(f"zStandard successfully installed/updated. It took {str(round(time.time()-startSec, 2))} seconds.") - return - - # Tell the user, but this is a best effort, so if it fails we don't care. - # Any user who wants to use RTSP and doesn't have ffmpeg installed can use our help docs to install it. - Logger.Debug(f"We didn't install zstandard. It took {str(round(time.time()-startSec, 2))} seconds. Output: {result.stderr}") - except Exception as e: - Logger.Debug(f"Error installing zstandard. {str(e)}") From 70cc9961d053c593281c0d568592d48a8a30c85e Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Mon, 24 Jun 2024 22:23:13 -0700 Subject: [PATCH 110/328] Adding the start of the logic for Bambu Cloud connecting --- .vscode/settings.json | 3 + bambu_octoeverywhere/bambuclient.py | 107 +++++++++-- bambu_octoeverywhere/bambucloud.py | 280 ++++++++++++++++++++++++++++ bambu_octoeverywhere/bambuhost.py | 4 + linux_host/config.py | 9 +- requirements.txt | 5 +- 6 files changed, 390 insertions(+), 18 deletions(-) create mode 100644 bambu_octoeverywhere/bambucloud.py diff --git a/.vscode/settings.json b/.vscode/settings.json index 9a968e6..1c57475 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -12,8 +12,10 @@ "backoff", "Bambu", "bambuclient", + "bambucloud", "bambucommandhandler", "bambuhost", + "bambulab", "bambumodels", "bambustatetranslater", "bambuwebcamhelper", @@ -50,6 +52,7 @@ "dnspython", "esac", "faststart", + "Fernet", "filamentchange", "filemetadatacache", "finalsnap", diff --git a/bambu_octoeverywhere/bambuclient.py b/bambu_octoeverywhere/bambuclient.py index c287c89..8753dbb 100644 --- a/bambu_octoeverywhere/bambuclient.py +++ b/bambu_octoeverywhere/bambuclient.py @@ -13,8 +13,18 @@ from linux_host.config import Config from linux_host.networksearch import NetworkSearch +from .bambucloud import BambuCloud, LoginStatus from .bambumodels import BambuState, BambuVersion + +class ConnectionContext: + def __init__(self, isCloud:bool, ipOrHostname:str, userName:str, accessToken:str): + self.IsCloud = isCloud + self.IpOrHostname = ipOrHostname + self.UserName = userName + self.AccessToken = accessToken + + # Responsible for connecting to and maintaining a connection to the Bambu Printer. # Also responsible for dispatching out MQTT update messages. class BambuClient: @@ -50,9 +60,10 @@ def __init__(self, logger:logging.Logger, config:Config, stateTranslator) -> Non # Get the required args. self.Config = config self.PortStr = config.GetStr(Config.SectionCompanion, Config.CompanionKeyPort, None) - self.AccessToken = config.GetStr(Config.SectionBambu, Config.BambuAccessToken, None) + self.LanAccessCode = config.GetStr(Config.SectionBambu, Config.BambuAccessToken, None) self.PrinterSn = config.GetStr(Config.SectionBambu, Config.BambuPrinterSn, None) - if self.PortStr is None or self.AccessToken is None or self.PrinterSn is None: + # The port and SN are required, but the Access Code isn't, since sometimes it's not there for cloud connections. + if self.PortStr is None or self.PrinterSn is None: raise Exception("Missing required args from the config") # We use this var to keep track of consecutively failed connections @@ -103,9 +114,6 @@ def _ClientWorker(self): # We always connect locally. We use encryption, but the printer doesn't have a trusted # cert root, so we have to disable the cert root checks. self.Client = mqtt.Client(mqtt.CallbackAPIVersion.VERSION2) - self.Client.tls_set(tls_version=ssl.PROTOCOL_TLS, cert_reqs=ssl.CERT_NONE) - self.Client.tls_insecure_set(True) - self.Client.username_pw_set("bblp", self.AccessToken) # Since we are local, we can do more aggressive reconnect logic. # The default is min=1 max=120 seconds. @@ -119,12 +127,24 @@ def _ClientWorker(self): self.Client.on_log = self._OnLog # Get the IP to try on this connect - ipOrHostname = self._GetIpOrHostnameToTry() + connectionContext = self._GetConnectionContextToTry() + if connectionContext.IsCloud: + self.Logger.info("Trying to connect to printer via Bambu Cloud...") + # We are connecting to Bambu Cloud, setup MQTT for it. + self.Client.tls_set(tls_version=ssl.PROTOCOL_TLS) + else: + # We are trying to connect to the printer locally, so configure mqtt for a LAN connection. + self.Logger.info("Trying to connect to printer via LAN...") + self.Client.tls_set(tls_version=ssl.PROTOCOL_TLS, cert_reqs=ssl.CERT_NONE) + self.Client.tls_insecure_set(True) + + # Set the username and access token. + self.Client.username_pw_set(connectionContext.UserName, connectionContext.AccessToken) # Connect to the server # This will throw if it fails, but after that, the loop_forever will handle reconnecting. localBackoffCounter += 1 - self.Client.connect(ipOrHostname, int(self.PortStr), keepalive=5) + self.Client.connect(connectionContext.IpOrHostname, int(self.PortStr), keepalive=5) # Note that self.Client.connect will not throw if there's no MQTT server, but not if auth is wrong. # So if it didn't throw, we know there's a server there, but it might not be the right server @@ -185,6 +205,14 @@ def _CleanupStateOnDisconnect(self): self.HasDoneFirstFullStateSync = False self.ReportSubscribeMid = None self.IsPendingSubscribe = False + # For some reason, the Bambu Cloud MQTT server will fire a disconnect message but doesn't actually disconnect. + # So we always call disconnect to ensure we force it, to ensure our connection loop closes. + try: + c = self.Client + if c is not None: + c.disconnect() + except Exception as e: + self.Logger.debug(f"_CleanupStateOnDisconnect exception on mqtt disconnect during cleanup. {e}") # Fired when the MQTT connection is made. @@ -342,24 +370,35 @@ def _Publish(self, msg:dict) -> bool: return False - # Gets the IP or hostname that should be used for the next connection attempt. - def _GetIpOrHostnameToTry(self) -> str: + # Returns a connection context object we should try to for this connection attempt. + # The connection context can indicate we are trying to connect to the Bambu Cloud or the local printer, + # depending on the plugin config and what's available. + def _GetConnectionContextToTry(self) -> ConnectionContext: # Increment and reset if it's too high. + # This will restart the process of trying cloud connect and falling back. self.ConsecutivelyFailedConnectionAttempts += 1 - if self.ConsecutivelyFailedConnectionAttempts > 5: + if self.ConsecutivelyFailedConnectionAttempts > 6: self.ConsecutivelyFailedConnectionAttempts = 0 - # On the first few attempts, use the expected IP. + # On the first few attempts, use the expected IP or the cloud config. # The first attempt will always be attempt 1, since it's reset to 0 and incremented before connecting. # The IP can be empty, like if the docker container is used, in which case we should always search for the printer. configIpOrHostname = self.Config.GetStr(Config.SectionCompanion, Config.CompanionKeyIpOrHostname, None) - if configIpOrHostname is not None and len(configIpOrHostname) > 0 and self.ConsecutivelyFailedConnectionAttempts < 3: - return configIpOrHostname + if self.ConsecutivelyFailedConnectionAttempts < 4: + # If we are using a Bambu cloud connection, try to return a connection object for it. + # We always try to do this for the first few attempts, since if it's setup as a Cloud connection, a LAN connection most likely won't work. + cloudContext = self._TryToGetCloudConnectContext() + if cloudContext is not None: + return cloudContext + + # If we aren't using a cloud connection or it failed, return the LAN hostname + if configIpOrHostname is not None and len(configIpOrHostname) > 0: + return self._GetLanConnectionContext(configIpOrHostname) # If we fail too many times, try to scan for the printer on the local subnet, the IP could have changed. # Since we 100% identify the printer by the access token and printer SN, we can try to scan for it. self.Logger.info(f"Searching for your Bambu Lab printer {self.PrinterSn}") - ips = NetworkSearch.ScanForInstances_Bambu(self.Logger, self.AccessToken, self.PrinterSn) + ips = NetworkSearch.ScanForInstances_Bambu(self.Logger, self.LanAccessCode, self.PrinterSn) # If we get an IP back, it is the printer. # The scan above will only return an IP if the printer was successfully connected to, logged into, and fully authorized with the Access Token and Printer SN. @@ -369,10 +408,46 @@ def _GetIpOrHostnameToTry(self) -> str: ip = ips[0] self.Logger.info(f"We found a new IP for this printer. [{configIpOrHostname} -> {ip}] Updating the config and using it to connect.") self.Config.SetStr(Config.SectionCompanion, Config.CompanionKeyIpOrHostname, ip) - return ip + return self._GetLanConnectionContext(ip) # If we don't find anything, just use the config IP. - return configIpOrHostname + return self._GetLanConnectionContext(configIpOrHostname) + + + def _GetLanConnectionContext(self, ipOrHostname) -> ConnectionContext: + # The username is always the same, we use the local LAN access token. + return ConnectionContext(False, ipOrHostname, "bblp", self.LanAccessCode) + + + # Returns a Bambu Cloud based connection context if it can be made, otherwise None + def _TryToGetCloudConnectContext(self) -> ConnectionContext: + bCloud = BambuCloud.Get() + if bCloud.HasContext() is False: + return None + + # Try to login and get the access token. + # Force the login to ensure the access token is current. + accessTokenResult = BambuCloud.Get().GetAccessToken(forceLogin=True) + + # If we failed, make sure to log the reason, so it's obvious for the user. + if accessTokenResult.Status != LoginStatus.Success: + self.Logger.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + self.Logger.error(" Failed To Log Into Bambu Cloud") + if accessTokenResult.Status == LoginStatus.BadUserNameOrPassword: + self.Logger.error("The email address or password is wrong. Re-run the Bambu Connect installer or use the docker files to update the email address and password.") + elif accessTokenResult.Status == LoginStatus.TwoFactorAuthEnabled: + self.Logger.error("To factor auth is enabled on this account. Bambu Lab doesn't allow us to support two factor auth, so it must be disabled on your account or LAN Only mode must be used on the printer.") + else: + self.Logger.error("Unknown error, we will try again later.") + self.Logger.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + + # We do a delay here, so we don't pound on the service. If we can't login for one of these reasons, we probably can't recover. + time.sleep(60.0 ^ self.ConsecutivelyFailedConnectionAttempts) + return None + + # Return the connection object. + accessToken = accessTokenResult.AccessToken + return ConnectionContext(True, bCloud.GetMqttHostname(), bCloud.GetUserNameFromAccessToken(accessToken), accessToken) # A class returned as the result of all commands. diff --git a/bambu_octoeverywhere/bambucloud.py b/bambu_octoeverywhere/bambucloud.py new file mode 100644 index 0000000..d09fa1d --- /dev/null +++ b/bambu_octoeverywhere/bambucloud.py @@ -0,0 +1,280 @@ +import json +import base64 +import logging +import threading +from enum import Enum + +import requests +from cryptography.fernet import Fernet + +from linux_host.config import Config + +from octoeverywhere.sentry import Sentry + + +# The result of a login request. +class LoginStatus(Enum): + Success = 0 # This is the only successful value + TwoFactorAuthEnabled = 1 + BadUserNameOrPassword = 2 + UnknownError = 3 + + +# The result of a get access token request. +# If the token is None, the Status will indicate why. +class AccessTokenResult(): + def __init__(self, status:LoginStatus, token:str = None) -> None: + self.Status = status + self.AccessToken = token + + +# A class that interacts with the Bambu Cloud. +# This github has a community made API docs: +# https://github.com/Doridian/OpenBambuAPI/blob/main/cloud-http.md +class BambuCloud: + + _Instance = None + + + @staticmethod + def Init(logger:logging.Logger, config:Config): + BambuCloud._Instance = BambuCloud(logger, config) + + + @staticmethod + def Get(): + return BambuCloud._Instance + + + def __init__(self, logger:logging.Logger, config:Config) -> None: + self.Logger = logger + self.Config = config + self.AccessToken = None + + + # Logs in given the user name and password. This doesn't support two factor auth at this time. + # Returns true if the login was successful, otherwise false. + def Login(self) -> LoginStatus: + try: + # Some notes on login. We were going to originally going to cache the access token and refresh token, so we didn't have to store the user name and password. + # However, the refresh token has an expiration on it, so eventually the user would have to re-enter their password, which isn't ideal. + # We also don't gain anything by storing the access token, since we need to hit an API anyways to make sure it's still valid and working. + self.Logger.info("Logging into Bambu Cloud...") + + # Get the correct URL. + url = self._GetBambuCloudApi("/v1/user-service/user/login") + + # Get the context. + email, password = self._GetContext() + if email is None or password is None: + self.Logger.error("Login Bambu Cloud failed to get context from the config.") + return LoginStatus.BadUserNameOrPassword + + # Make the request. + response = requests.post(url, json={'account': email, 'password': password}, timeout=30) + + # Check the response. + if response.status_code != 200: + if response.status_code == 400: + self.Logger.error("Login Bambu Cloud failed with status code: 400 bad request. The user name or password are probably wrong or has changed.") + return LoginStatus.BadUserNameOrPassword + self.Logger.error(f"Login Bambu Cloud failed with status code: {response.status_code}") + return LoginStatus.UnknownError + + # If the user has two factor auth enabled, this will still return 200, but there will be a tfaKey field with a string. + j = response.json() + tfaKey = j.get('tfaKey', None) + if tfaKey is not None and len(tfaKey) > 0: + self.Logger.error("Login Bambu Cloud failed because two factor auth is enabled. Bambu Lab's APIs don't allow us to support two factor at this time.") + return LoginStatus.TwoFactorAuthEnabled + + # Try to get the access token + accessToken = j.get('accessToken', None) + if accessToken is None or len(accessToken) == 0: + self.Logger.error("Login Bambu Cloud failed because access token was not found in the response.") + return LoginStatus.UnknownError + self.AccessToken = accessToken + + # The token expiration is usually 1 year, we just check it for now. + expiresIn = int(j.get('expiresIn', 0)) + if expiresIn / 60 / 60 / 24 < 300: + self.Logger.warn(f"Login Bambu Cloud access token expires in {expiresIn} seconds") + + # Every time we login in, we also want to ensure the printer's cloud info is synced locally. + # Right now this can only sync the access code, but this is important, because things like the webcam streaming need to know the access code. + self.SyncBambuCloudInfoAsync() + + # Success + return LoginStatus.Success + + except Exception as e: + Sentry.Exception("Bambu Cloud login exception", e) + return LoginStatus.UnknownError + + + # Returns the access token. + # If there's no valid access token, this will try a blocking login. + def GetAccessToken(self, forceLogin = False) -> AccessTokenResult: + # If we already have the access token, we are good. + if forceLogin is False and self.AccessToken is not None and len(self.AccessToken) > 0: + return AccessTokenResult(LoginStatus.Success, self.AccessToken) + + # Else, try a login. + status = self.Login() + return AccessTokenResult(status, self.AccessToken) + + + # Used to clear the access token if there's a failure using it. + def _ResetAccessToken(self): + self.AccessToken = None + + + # A helper to decode the access token and get the Bambu Cloud username. + # Returns None on failure. + def GetUserNameFromAccessToken(self, accessToken: str) -> str: + try: + # The Access Token is a JWT, we need the second part to decode. + accountInfoBase64 = accessToken.split(".")[1] + # The string len must be a multiple of 4, padded with "=" + while (len(accountInfoBase64)) % 4 != 0: + accountInfoBase64 += "=" + # Decode and parse as json. + jsonAuthToken = json.loads(base64.b64decode(accountInfoBase64)) + return jsonAuthToken["username"] + except Exception as e: + Sentry.Exception("Bambu Cloud GetUserNameFromAccessToken exception", e) + return None + + + # Returns a list of the user's devices. + # Returns None on failure. + # Special Note: This function is used as a access token validation check. So if this fails due to the access token being invalid, the access token should be cleared so we try to login again. + def GetDeviceList(self) -> dict: + tokenResult = self.GetAccessToken() + if tokenResult.Status != LoginStatus.Success: + return None + + # Get the API + url = self._GetBambuCloudApi("/v1/iot-service/api/user/bind") + + # Make the request. + headers = {'Authorization': 'Bearer ' + tokenResult.AccessToken} + response = requests.get(url, headers=headers, timeout=10) + if response.status_code != 200: + self.Logger.error(f"Bambu Cloud GetDeviceList failed with status code: {response.status_code}") + # On failure reset the access token. + self._ResetAccessToken() + return None + self.Logger.debug(f"Bambu Cloud Device List: {response.json()}") + devices = response.json().get('devices', None) + if devices is None: + self.Logger.error("Bambu Cloud GetDeviceList failed, the devices object was missing.") + return None + return response.json()['devices'] + + + # Returns this device info from the Bambu Cloud API by matching the SN + def GetThisDeviceInfo(self) -> dict: + devices = self.GetDeviceList() + localSn = self.Config.GetStr(Config.SectionBambu, Config.BambuPrinterSn, None) + if localSn is None: + self.Logger.error("Bambu Cloud GetThisDeviceInfo has no local printer SN to match.") + return None + for d in devices: + sn = d.get('dev_id', None) + self.Logger.debug(f"Bambu Cloud Printer Info. SN:{sn} Name:{(d.get('name', None))}") + if sn == localSn: + return d + self.Logger.error("Bambu Cloud failed to find a matching printer SN on the user account.") + return None + + + # Get's the known device info from the Bambu API and ensures it's synced with our config settings. + def SyncBambuCloudInfoAsync(self) -> bool: + threading.Thread(target=self.SyncBambuCloudInfo, daemon=True).start() + + + def SyncBambuCloudInfo(self) -> bool: + try: + info = self.GetThisDeviceInfo() + if info is None: + self.Logger.error("Bambu Cloud SyncBambuCloudInfo didn't find printer info.") + return False + accessCode = info.get('dev_access_code', None) + if accessCode is None: + self.Logger.error("Bambu Cloud SyncBambuCloudInfo didn't find an access code.") + return False + # Update the access code if it's changed. + if self.Config.GetStr(Config.SectionBambu, Config.BambuAccessToken, "") != accessCode: + self.Config.SetStr(Config.SectionBambu, Config.BambuAccessToken, accessCode) + self.Logger.info("Bambu Cloud SyncBambuCloudInfo updated the access code.") + return True + except Exception as e: + Sentry.Exception("SyncBambuCloudInfo exception", e) + return False + + + def _IsRegionChina(self) -> bool: + region = self.Config.GetStr(Config.SectionBambu, Config.BambuCloudRegion, None) + if region is None: + self.Logger.warn("Bambu Cloud region not set, assuming world wide.") + region = "world" + return region == "china" + + + # Returns the correct MQTT hostname depending on the region. + def GetMqttHostname(self): + if self._IsRegionChina(): + return "cn.mqtt.bambulab.com" + return "us.mqtt.bambulab.com" + + + # Returns the correct full API URL based on the region. + def _GetBambuCloudApi(self, urlPathAndParams:str): + if self._IsRegionChina(): + return "https://api.bambulab.cn" + urlPathAndParams + return "https://api.bambulab.com" + urlPathAndParams + + + # Sets the user's context into the config file. + def SetContext(self, email:str, p:str) -> bool: + try: + # This isn't ideal, but there's nothing we can do better locally on the device. + # So at least it's not just plain text. + data = {"email":email, "p":p} + j = json.dumps(data) + f = Fernet(b"iyqYOs9QPwO5J6jW30uPJIxywhf7yLrvaRXLp5gi9OA=") + token = f.encrypt(j.encode()) + self.Config.SetStr(Config.SectionBambu, Config.BambuCloudContext, token.decode()) + return True + except Exception as e: + Sentry.Exception("Bambu Cloud set email exception", e) + return False + + + # Returns if there's a user context in the config file. + # This doesn't check if the user context is valid, just that it's there. + def HasContext(self) -> bool: + (e, p) = self._GetContext() + return e is not None and p is not None + + + # Sets the user's context from the config file. + def _GetContext(self): + try: + token = self.Config.GetStr(Config.SectionBambu, Config.BambuCloudContext, None) + if token is None: + self.Logger.error("No Bambu Cloud context found in the config file.") + return (None, None) + f = Fernet(b"iyqYOs9QPwO5J6jW30uPJIxywhf7yLrvaRXLp5gi9OA=") + jsonStr = f.decrypt(token.encode()) + data = json.loads(jsonStr) + e = data.get("email", None) + p = data.get("p", None) + if e is None or p is None: + self.Logger.error("No Bambu Cloud context was missing required data.") + return (None, None) + return (e, p) + except Exception as e: + Sentry.Exception("Bambu Cloud login exception", e) + return (None, None) diff --git a/bambu_octoeverywhere/bambuhost.py b/bambu_octoeverywhere/bambuhost.py index 104ecbe..955d601 100644 --- a/bambu_octoeverywhere/bambuhost.py +++ b/bambu_octoeverywhere/bambuhost.py @@ -21,6 +21,7 @@ from linux_host.version import Version from linux_host.logger import LoggerInit +from .bambucloud import BambuCloud from .bambuclient import BambuClient from .bambuwebcamhelper import BambuWebcamHelper from .bambucommandhandler import BambuCommandHandler @@ -132,6 +133,9 @@ def RunBlocking(self, configPath, localStorageDir, repoRoot, devConfig_CanBeNone # Setup the command handler CommandHandler.Init(self.Logger, self.NotificationHandler, BambuCommandHandler(self.Logger)) + # Setup the cloud if it's setup in the config. + BambuCloud.Init(self.Logger, self.Config) + # Setup and start the Bambu Client BambuClient.Init(self.Logger, self.Config, stateTranslator) diff --git a/linux_host/config.py b/linux_host/config.py index 17eee26..2c2c0de 100644 --- a/linux_host/config.py +++ b/linux_host/config.py @@ -56,6 +56,9 @@ class Config: SectionBambu = "bambu" BambuAccessToken = "access_token" BambuPrinterSn = "printer_serial_number" + # Used if the user is logged into Bambu Cloud + BambuCloudContext = "cloud_context" + BambuCloudRegion = "cloud_region" # This allows us to add comments into our config. @@ -240,9 +243,13 @@ def SetStr(self, section, key, value) -> None: if self.Config.has_section(section) is False: self.Config.add_section(section) if value is None: - # If we are setting to None, delete the key if it exists. + # None is a special case, if we are setting it, delete the key if it exists. if key in self.Config[section].keys(): del self.Config[section][key] + else: + # If there was no key, return early, since we did nothing. + # This is a common case, since we use GetStr(..., ..., None) often to get the value if it exists. + return else: # If not none, set the key self.Config[section][key] = value diff --git a/requirements.txt b/requirements.txt index 1d03d2c..e601644 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,7 +18,10 @@ httpx>=0.24.1,<0.26.0 urllib3>=1.26.15,<2.0.0 sentry-sdk>=1.19.1,<2 #zstandard>=0.22.0,<0.23.0 + # The following are required only for Moonraker configparser + # Only used for Bambu Connect -paho-mqtt>=2.0.0 \ No newline at end of file +paho-mqtt>=2.0.0 +cryptography>=42.0.8 \ No newline at end of file From 7c25222166ad26847a6db5023de28c47f082aa5b Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Tue, 25 Jun 2024 20:38:34 -0700 Subject: [PATCH 111/328] Updating the Bambu Connect docker image for the new Bambu Cloud setup. --- bambu_octoeverywhere/bambuclient.py | 10 +-- bambu_octoeverywhere/bambucloud.py | 18 ++-- docker-compose.yml | 30 +++++-- docker-readme.md | 56 ++++++++++-- docker_octoeverywhere/__main__.py | 132 +++++++++++++++++++++------- 5 files changed, 193 insertions(+), 53 deletions(-) diff --git a/bambu_octoeverywhere/bambuclient.py b/bambu_octoeverywhere/bambuclient.py index 8753dbb..a86fad0 100644 --- a/bambu_octoeverywhere/bambuclient.py +++ b/bambu_octoeverywhere/bambuclient.py @@ -431,18 +431,18 @@ def _TryToGetCloudConnectContext(self) -> ConnectionContext: # If we failed, make sure to log the reason, so it's obvious for the user. if accessTokenResult.Status != LoginStatus.Success: - self.Logger.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") - self.Logger.error(" Failed To Log Into Bambu Cloud") + self.Logger.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + self.Logger.error(" Failed To Log Into Bambu Cloud") if accessTokenResult.Status == LoginStatus.BadUserNameOrPassword: - self.Logger.error("The email address or password is wrong. Re-run the Bambu Connect installer or use the docker files to update the email address and password.") + self.Logger.error("The email address or password is wrong. Re-run the Bambu Connect installer or use the docker files to update your email address and password.") elif accessTokenResult.Status == LoginStatus.TwoFactorAuthEnabled: self.Logger.error("To factor auth is enabled on this account. Bambu Lab doesn't allow us to support two factor auth, so it must be disabled on your account or LAN Only mode must be used on the printer.") else: self.Logger.error("Unknown error, we will try again later.") - self.Logger.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + self.Logger.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") # We do a delay here, so we don't pound on the service. If we can't login for one of these reasons, we probably can't recover. - time.sleep(60.0 ^ self.ConsecutivelyFailedConnectionAttempts) + time.sleep(600.0 * self.ConsecutivelyFailedConnectionAttempts) return None # Return the connection object. diff --git a/bambu_octoeverywhere/bambucloud.py b/bambu_octoeverywhere/bambucloud.py index d09fa1d..98939c4 100644 --- a/bambu_octoeverywhere/bambucloud.py +++ b/bambu_octoeverywhere/bambucloud.py @@ -65,7 +65,7 @@ def Login(self) -> LoginStatus: url = self._GetBambuCloudApi("/v1/user-service/user/login") # Get the context. - email, password = self._GetContext() + email, password = self.GetContext() if email is None or password is None: self.Logger.error("Login Bambu Cloud failed to get context from the config.") return LoginStatus.BadUserNameOrPassword @@ -75,10 +75,15 @@ def Login(self) -> LoginStatus: # Check the response. if response.status_code != 200: + body = "" + try: + body = json.dumps(response.json()) + except Exception: + pass if response.status_code == 400: - self.Logger.error("Login Bambu Cloud failed with status code: 400 bad request. The user name or password are probably wrong or has changed.") + self.Logger.error(f"Login Bambu Cloud failed with status code: 400 bad request. The user name or password are probably wrong or has changed. Response: {body}") return LoginStatus.BadUserNameOrPassword - self.Logger.error(f"Login Bambu Cloud failed with status code: {response.status_code}") + self.Logger.error(f"Login Bambu Cloud failed with status code: {response.status_code}, Response: {body}") return LoginStatus.UnknownError # If the user has two factor auth enabled, this will still return 200, but there will be a tfaKey field with a string. @@ -255,16 +260,17 @@ def SetContext(self, email:str, p:str) -> bool: # Returns if there's a user context in the config file. # This doesn't check if the user context is valid, just that it's there. def HasContext(self) -> bool: - (e, p) = self._GetContext() + (e, p) = self.GetContext() return e is not None and p is not None # Sets the user's context from the config file. - def _GetContext(self): + def GetContext(self, expectContextToExist = True): try: token = self.Config.GetStr(Config.SectionBambu, Config.BambuCloudContext, None) if token is None: - self.Logger.error("No Bambu Cloud context found in the config file.") + if expectContextToExist: + self.Logger.error("No Bambu Cloud context found in the config file.") return (None, None) f = Fernet(b"iyqYOs9QPwO5J6jW30uPJIxywhf7yLrvaRXLp5gi9OA=") jsonStr = f.decrypt(token.encode()) diff --git a/docker-compose.yml b/docker-compose.yml index 653737e..bc40a9b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,26 +3,44 @@ services: octoeverywhere-bambu-connect: image: octoeverywhere/octoeverywhere:latest environment: - # https://octoeverywhere.com/s/access-code - - ACCESS_CODE=XXXXXXXX # https://octoeverywhere.com/s/bambu-sn - SERIAL_NUMBER=XXXXXXXXXXXXXXX # Find using the printer's display - PRINTER_IP=192.168.1.1 + + # ~~~ If connecting with Bambu Cloud Mode ~~~ + # https://octoeverywhere.com/s/bambu-setup + - BAMBU_CLOUD_ACCOUNT_EMAIL=XXXXXXXX + - BAMBU_CLOUD_ACCOUNT_PASSWORD=XXXXXXXX + #- BAMBU_CLOUD_REGION=china # Optional, use if your Bambu account is in the China region + + # ~~~ OR If connecting with LAN Only Mode ~~~ + # https://octoeverywhere.com/s/access-code + # - ACCESS_CODE=XXXXXXXX + # - LAN_ONLY_MODE=TRUE volumes: # Specify a path mapping for the required persistent storage - - /some/path/on/your/computer:/data + - /local/computer/data/folder:/data # Add as many printers as you want! # octoeverywhere-bambu-connect-2: # image: octoeverywhere/octoeverywhere:latest # environment: - # # https://octoeverywhere.com/s/access-code - # - ACCESS_CODE=XXXXXXXX # # https://octoeverywhere.com/s/bambu-sn # - SERIAL_NUMBER=XXXXXXXXXXXXXXX # # Find using the printer's display # - PRINTER_IP=192.168.1.2 + + # # ~~~ If connecting with Bambu Cloud Mode ~~~ + # # https://octoeverywhere.com/s/bambu-setup + # - BAMBU_CLOUD_ACCOUNT_EMAIL=XXXXXXXX + # - BAMBU_CLOUD_ACCOUNT_PASSWORD=XXXXXXXX + # #- BAMBU_CLOUD_REGION=china # Optional, use if your Bambu account is in the China region + + # # ~~~ OR If connecting with LAN Only Mode ~~~ + # # https://octoeverywhere.com/s/access-code + # # - ACCESS_CODE=XXXXXXXX + # # - LAN_ONLY_MODE=TRUE # volumes: # # Specify a path mapping for the required persistent storage - # - /some/path/on/your/computer/printer2:/data \ No newline at end of file + # - /local/computer/data/folder2:/data \ No newline at end of file diff --git a/docker-readme.md b/docker-readme.md index a43a7fb..6b686e9 100644 --- a/docker-readme.md +++ b/docker-readme.md @@ -4,19 +4,62 @@ OctoEverywhere's docker image only works for [Bambu Connect](https://octoeverywh Official Docker Image: https://hub.docker.com/r/octoeverywhere/octoeverywhere + +## Bambu Cloud Vs Lan Only Connection Modes + +Bambu Labs made a change where 3rd-party addons can't connect to the printer directly over your LAN network if the printer is connected to Bambu Cloud. + +You have two options to setup OctoEverywhere: + +1) Connect OctoEverywhere via the Bambu Cloud +2) Put your printer in Lan Only Mode and connect locally + + +### Connect Via Bambu Cloud + +If you want to continue using Bambu Cloud, you can set up OctoEverywhere to access your printer via Bambu Cloud. To do so, you simply need to provide the docker image with your Bambu Cloud account email address and password. + +**Rest assured, your Bambu Cloud email address and password are stored locally, encrypted on disk, and are never sent to the OctoEverywhere service.** + +Additionally, due to restrictions from Bambu Labs, no 3rd-party services can support accounts with two-factor authentication enabled. Two-factor authentication must be disabled on your account to use OctoEverywhere; use a strong password instead. + +If your Bambu Cloud account is setup to login with Google, Apple, or another 3rd party login service, you need to set an account password: +- Login to the Bambu Handy mobile app using the 3rd party provider. +- Tap the person icon at the bottom right. +- Tap Account Security > Change Password. + +This will allow you to set a password. You can then use your email address and password with Bambu Connect. + +### Use LAN Only Mode + +If you don't mind disabling the Bambu Cloud, you can enable "LAN only mode" on your Bambu Lab 3D printer. With Bambu Cloud disabled, you WILL still be able to use Bambu Studio and Bambu Handy while on the same network as your 3D printer. OctoEverywhere can then directly connect to your printer over you local network, there's no need to supply a Bambu Cloud email or password. + ## Required Setup Environment Vars To use the Bambu Connect plugin, you need to get the following information. -- Your printer's Access Code - https://octoeverywhere.com/s/access-code - Your printer's Serial Number - https://octoeverywhere.com/s/bambu-sn - Your printer's IP Address - (use the printer's display) +- If you're connecting with Bambu Cloud: + - Your Bambu Cloud account email address + - Your Bambu Cloud account password + - Note: Your Bambu Cloud email address and password are stored locally, encrypted on disk, and never sent to the OctoEverywhere service. + - Learn more here - https://octoeverywhere.com/s/bambu-setup +- If you're connecting in LAN Only Mode: + - Your printer's Access Code - https://octoeverywhere.com/s/access-code -These three values must be set at environment vars when you first run the container. Once the container is run, you don't need to include them again, unless you want to update the values. +These three values must be set at environment vars when you first run the container. Once the container is ran, you don't need to include them, unless you want to update the values. -- ACCESS_CODE=(code) - SERIAL_NUMBER=(serial number) - PRINTER_IP=(ip address) +- If connecting via Bambu Cloud: + - BAMBU_CLOUD_ACCOUNT_EMAIL=(email) + - BAMBU_CLOUD_ACCOUNT_PASSWORD=(password) + - Optional - BAMBU_CLOUD_REGION=china - Use if your Bambu account is in the China region. +- If connecting via LAN Only Mode: + - ACCESS_CODE=(code) + - LAN_ONLY_MODE=TRUE + ## Required Persistent Storage @@ -51,12 +94,13 @@ Docker compose is a fancy wrapper to run docker containers. You can also run doc Use a command like this example, but update the required vars. `docker pull octoeverywhere/octoeverywhere` -`docker run --name bambu-connect -e ACCESS_CODE= -e SERIAL_NUMBER= -e PRINTER_IP= -v /your/local/path:/data -d octoeverywhere/octoeverywhere` + +`docker run --name bambu-connect -e SERIAL_NUMBER= -e PRINTER_IP= -e BAMBU_CLOUD_ACCOUNT_EMAIL="" -e BAMBU_CLOUD_ACCOUNT_PASSWORD="" -v /your/local/data/folder:/data -d octoeverywhere/octoeverywhere` Follow the "Linking Your Bambu Connect Plugin" to link the plugin to your account. ## Building The Image Locally -You can build the docker image locally if you prefer, use the following command. +You can build the docker image locally if you prefer; use the following command. -`docker build -t octoeverywhere .` \ No newline at end of file +`docker build -t octoeverywhere-local .` \ No newline at end of file diff --git a/docker_octoeverywhere/__main__.py b/docker_octoeverywhere/__main__.py index 863123a..ad0cf7f 100644 --- a/docker_octoeverywhere/__main__.py +++ b/docker_octoeverywhere/__main__.py @@ -16,6 +16,8 @@ from linux_host.startup import Startup from linux_host.config import Config +from bambu_octoeverywhere.bambucloud import BambuCloud + # pylint: disable=logging-fstring-interpolation if __name__ == '__main__': @@ -70,28 +72,9 @@ def CreateDirIfNotExists(path: str) -> None: logger.info(f"Init config object: {configPath}") config = Config(configPath) - # If there is a arg passed, always update or set it. - # This allows users to update the values after the image has ran the first time. - accessCode = os.environ.get("ACCESS_CODE", None) - if accessCode is not None: - logger.info(f"Setting Access Code: {accessCode}") - config.SetStr(Config.SectionBambu, Config.BambuAccessToken, accessCode) - # Ensure something is set now. - if config.GetStr(Config.SectionBambu, Config.BambuAccessToken, None) is None: - logger.error("") - logger.error("") - logger.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") - logger.error(" You must pass the printer's Access Code as an env var.") - logger.error(" Use `docker run -e ACCESS_CODE=` or add it to your docker-compose file.") - logger.error("") - logger.error(" To find your Access Code -> https://octoeverywhere.com/s/access-code") - logger.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") - logger.error("") - logger.error("") - # Sleep some, so we don't restart super fast and then exit. - time.sleep(5.0) - sys.exit(1) + # The serial number is always required, in both Bambu Cloud and LAN mode. + # So we always get that first. printerSn = os.environ.get("SERIAL_NUMBER", None) if printerSn is not None: logger.info(f"Setting Serial Number: {printerSn}") @@ -112,14 +95,103 @@ def CreateDirIfNotExists(path: str) -> None: time.sleep(5.0) sys.exit(1) - # - # If we got here, the access token and serial number are set or were already set. - # We should be able to launch! - # + # Bambu updated the printer and broke LAN access unless the printer is in LAN mode. + # The work around was to connect to the Bambu Cloud instead of directly to the printer. + # The biggest downside of this is that we need to get the user's email address and password for Bambu Cloud. + # BUT the user can also do the LAN only mode, if they want to. + isLanOnlyMode = os.environ.get("LAN_ONLY_MODE", False) + isAccessCodeRequired = True + if isLanOnlyMode: + # In LAN only mode we only need the Serial number and access code. + logger.info("Connection Mode: LAN Only (Use the env var LAN_ONLY_MODE=FALSE to enable Bambu Cloud mode.)") + # This is LAN only mode, where we need the user to get us the Access Code. (In the cloud mode, we can get it from the Bambu Cloud API) + # If there is a arg passed, always update or set it. + # This allows users to update the values after the image has ran the first time. + accessCode = os.environ.get("ACCESS_CODE", None) + if accessCode is not None: + logger.info(f"Setting Access Code: {accessCode}") + config.SetStr(Config.SectionBambu, Config.BambuAccessToken, accessCode) + # Ensure something is set now. + if config.GetStr(Config.SectionBambu, Config.BambuAccessToken, None) is None: + logger.error("") + logger.error("") + logger.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + logger.error(" You must pass the printer's Access Code as an env var.") + logger.error(" Use `docker run -e ACCESS_CODE=` or add it to your docker-compose file.") + logger.error("") + logger.error(" To find your Access Code -> https://octoeverywhere.com/s/access-code") + logger.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + logger.error("") + logger.error("") + # Sleep some, so we don't restart super fast and then exit. + time.sleep(5.0) + sys.exit(1) + else: + logger.info("Connection Mode: Bambu Cloud (Use the env var LAN_ONLY_MODE=TRUE to enable LAN Only mode.)") + # In Bambu Cloud mode, we need the user's email and password. + bambuCloud = BambuCloud(logger, config) + # Get any existing values. + (bambuCloudEmail, bambuCloudPassword) = bambuCloud.GetContext(expectContextToExist=False) + bambuCloudEmail = os.environ.get("BAMBU_CLOUD_ACCOUNT_EMAIL", bambuCloudEmail) + bambuCloudPassword = os.environ.get("BAMBU_CLOUD_ACCOUNT_PASSWORD", bambuCloudPassword) + + # Ensure the context is already set or the user passed the email and password. + if bambuCloudEmail is None: + logger.error("") + logger.error("") + logger.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + logger.error(" You must pass your Bambu Cloud account email address as an env var.") + logger.error("Use `docker run -e BAMBU_CLOUD_ACCOUNT_EMAIL=` or add it to your docker-compose file.") + logger.error("") + logger.error(" Your Bambu email address and password are KEPT LOCALLY, encrypted on disk") + logger.error(" and are NEVER SENT to the OctoEverywhere service.") + logger.error("") + logger.error(" For Help And More Details -> https://octoeverywhere.com/s/bambu-setup") + logger.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + logger.error("") + logger.error("") + # Sleep some, so we don't restart super fast and then exit. + time.sleep(5.0) + sys.exit(1) + if bambuCloudPassword is None: + logger.error("") + logger.error("") + logger.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + logger.error(" You must pass your Bambu Cloud account password as an env var.") + logger.error("Use `docker run -e BAMBU_CLOUD_ACCOUNT_PASSWORD=` or add it to your docker-compose file.") + logger.error("") + logger.error(" Your Bambu email address and password are KEPT LOCALLY, encrypted on disk") + logger.error(" and are NEVER SENT to the OctoEverywhere service.") + logger.error("") + logger.error(" For Help And More Details -> https://octoeverywhere.com/s/bambu-setup") + logger.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + logger.error("") + logger.error("") + # Sleep some, so we don't restart super fast and then exit. + time.sleep(5.0) + sys.exit(1) + + # Update the context now, since it might have changed. + logger.info(f"Setting Bambu Cloud Context: {bambuCloudEmail}") + if bambuCloud.SetContext(bambuCloudEmail, bambuCloudPassword) is False: + # This should never happen. If it does allow the setup to continue, but log the error. + logger.error("Failed to set the Bambu Cloud context.") + + # The region is optional. + bambuCloudRegion = os.environ.get("BAMBU_CLOUD_REGION", None) + if bambuCloudRegion is not None: + bambuCloudRegion = bambuCloudRegion.lower().trim() + if bambuCloudRegion != "china": + logger.warning("The BAMBU_CLOUD_REGION should only be set to 'china' if the account is in the China region. For all other accounts it should not be set.") + logger.info(f"Setting Bambu Cloud Region To: {bambuCloudRegion}") + config.SetStr(Config.SectionBambu, Config.BambuCloudRegion, bambuCloudRegion) + # Ensure something is set now. + if config.GetStr(Config.SectionBambu, Config.BambuCloudRegion, None) is None: + logger.info("Setting Bambu Cloud to the default value for world wide accounts.") + config.SetStr(Config.SectionBambu, Config.BambuCloudRegion, "worldwide") - # TEMP - Until we fix the issue where the plugin doesn't know the local LAN network address range, we need the - # user to pass the printer's IP to the plugin, since the auto scanning doesn't work. - # When this is fixed, we no longer need it to be passed. + # For now, we also need the user to supply the printer's IP address, since we can't auto scan the network in docker. + # We also need this for the Bambu Cloud mode, since we can't get it from the Bambu Cloud API and we can't scan for the printer. printerId = os.environ.get("PRINTER_IP", None) if printerId is not None: logger.info(f"Setting Printer IP: {printerId}") @@ -129,10 +201,10 @@ def CreateDirIfNotExists(path: str) -> None: logger.error("") logger.error("") logger.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") - logger.error(" You must pass the printer's IP Address as an env var.") + logger.error(" You must pass the printer's IP Address as an env var.") logger.error(" Use `docker run -e PRINTER_IP=` or add it to your docker-compose file.") logger.error("") - logger.error(" To find your Ip Address, use the display on your printer.") + logger.error(" To find your printer's IP Address -> https://octoeverywhere.com/s/bambu-ip") logger.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") logger.error("") logger.error("") From bda5e2dd2c768c1957b7ae41eaa66fcfa9135621 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Wed, 26 Jun 2024 21:07:30 -0700 Subject: [PATCH 112/328] Final updates to the docker and docker compose docs for Bambu Connect --- docker-compose.yml | 16 +++++----- docker-readme.md | 76 +++++++++++++++++++++------------------------- setup.py | 2 +- 3 files changed, 43 insertions(+), 51 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index bc40a9b..560a77e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,12 +1,11 @@ -version: '2' services: octoeverywhere-bambu-connect: image: octoeverywhere/octoeverywhere:latest environment: # https://octoeverywhere.com/s/bambu-sn - SERIAL_NUMBER=XXXXXXXXXXXXXXX - # Find using the printer's display - - PRINTER_IP=192.168.1.1 + # Find using the printer's display or use https://octoeverywhere.com/s/bambu-ip + - PRINTER_IP=XXX.XXX.XXX.XXX # ~~~ If connecting with Bambu Cloud Mode ~~~ # https://octoeverywhere.com/s/bambu-setup @@ -20,16 +19,17 @@ services: # - LAN_ONLY_MODE=TRUE volumes: # Specify a path mapping for the required persistent storage - - /local/computer/data/folder:/data + # This can also be an absolue path, e.g. /var/octoeverywhere/plugin/data or /c/users/name/plugin/data + - ./data:/data - # Add as many printers as you want! + # Add as many printers as you want! Just make the name `octoeverywhere-bambu-connect` unique! # octoeverywhere-bambu-connect-2: # image: octoeverywhere/octoeverywhere:latest # environment: # # https://octoeverywhere.com/s/bambu-sn # - SERIAL_NUMBER=XXXXXXXXXXXXXXX - # # Find using the printer's display - # - PRINTER_IP=192.168.1.2 + # # Find using the printer's display or use https://octoeverywhere.com/s/bambu-ip + # - PRINTER_IP=XXX.XXX.XXX.XXX # # ~~~ If connecting with Bambu Cloud Mode ~~~ # # https://octoeverywhere.com/s/bambu-setup @@ -43,4 +43,4 @@ services: # # - LAN_ONLY_MODE=TRUE # volumes: # # Specify a path mapping for the required persistent storage - # - /local/computer/data/folder2:/data \ No newline at end of file + # - ./data:/data \ No newline at end of file diff --git a/docker-readme.md b/docker-readme.md index 6b686e9..f9769df 100644 --- a/docker-readme.md +++ b/docker-readme.md @@ -1,73 +1,53 @@ # Bambu Connect Docker Support -OctoEverywhere's docker image only works for [Bambu Connect](https://octoeverywhere.com/bambu?source=github_docker_readme) for Bambu Lab 3D printers. If you are using OctoPrint or Klipper, [follow our getting started guide](https://octoeverywhere.com/getstarted?source=github_docker_readme) to install the OctoEverywhere plugin. +OctoEverywhere's docker image only works with [Bambu Connect](https://octoeverywhere.com/bambu?source=github_docker_readme) for Bambu Lab 3D printers. If you are using OctoPrint or Klipper, [follow our getting started guide to install the OctoEverywhere plugin.](https://octoeverywhere.com/getstarted?source=github_docker_readme) Official Docker Image: https://hub.docker.com/r/octoeverywhere/octoeverywhere ## Bambu Cloud Vs Lan Only Connection Modes -Bambu Labs made a change where 3rd-party addons can't connect to the printer directly over your LAN network if the printer is connected to Bambu Cloud. +Bambu Lab made a firmware change in July 2024 where 3rd-party addons can't connect to the printer directly over your LAN network if the printer is connected to Bambu Cloud. -You have two options to setup OctoEverywhere: +Thus, you can pick either of these install methods: -1) Connect OctoEverywhere via the Bambu Cloud -2) Put your printer in Lan Only Mode and connect locally +1) Connect OctoEverywhere to your 3D printer through Bambu Cloud. +2) Put your 3D printer in "LAN Only Mode" and connect OctoEverywhere locally to the 3D printer. +Note if you put your printer in "LAN Only Mode" you **can** still use Bambu Studio and Bambu Handy when on the same network as the 3D printer. ### Connect Via Bambu Cloud -If you want to continue using Bambu Cloud, you can set up OctoEverywhere to access your printer via Bambu Cloud. To do so, you simply need to provide the docker image with your Bambu Cloud account email address and password. +For OctoEverywhere to connect to your 3D printer through Bambu Cloud, you just need to supply your Bambu Cloud account info to the local plugin. **Rest assured, your Bambu Cloud email address and password are stored locally, encrypted on disk, and are never sent to the OctoEverywhere service.** -Additionally, due to restrictions from Bambu Labs, no 3rd-party services can support accounts with two-factor authentication enabled. Two-factor authentication must be disabled on your account to use OctoEverywhere; use a strong password instead. +If you use Facebook, Google, or Apple to login to Bambu Cloud, [follow this guide to set a password on your account.](https://intercom.help/octoeverywhere/en/articles/9529936-bambu-cloud-with-bambu-connect) -If your Bambu Cloud account is setup to login with Google, Apple, or another 3rd party login service, you need to set an account password: -- Login to the Bambu Handy mobile app using the 3rd party provider. -- Tap the person icon at the bottom right. -- Tap Account Security > Change Password. -This will allow you to set a password. You can then use your email address and password with Bambu Connect. +### Connect Via 'LAN Only Mode' -### Use LAN Only Mode +If you don't mind disabling the Bambu Cloud, you can enable "LAN only mode" on your Bambu Lab 3D printer. -If you don't mind disabling the Bambu Cloud, you can enable "LAN only mode" on your Bambu Lab 3D printer. With Bambu Cloud disabled, you WILL still be able to use Bambu Studio and Bambu Handy while on the same network as your 3D printer. OctoEverywhere can then directly connect to your printer over you local network, there's no need to supply a Bambu Cloud email or password. +In "LAN only mode" OctoEverywhere can directly connect to your 3D printer on you local network, there's no need to supply your Bambu Cloud email or password. With Bambu Cloud disabled, you WILL still be able to use Bambu Studio and Bambu Handy while on the same network as your 3D printer. -## Required Setup Environment Vars +## Required Setup Information To use the Bambu Connect plugin, you need to get the following information. - Your printer's Serial Number - https://octoeverywhere.com/s/bambu-sn -- Your printer's IP Address - (use the printer's display) -- If you're connecting with Bambu Cloud: +- Your printer's IP Address - Use the printer's display or https://octoeverywhere.com/s/bambu-ip +- If you're connecting with Bambu Cloud... - Your Bambu Cloud account email address - Your Bambu Cloud account password - - Note: Your Bambu Cloud email address and password are stored locally, encrypted on disk, and never sent to the OctoEverywhere service. - - Learn more here - https://octoeverywhere.com/s/bambu-setup -- If you're connecting in LAN Only Mode: + - **Note:** Your Bambu Cloud email address and password are stored locally, encrypted on disk, and never sent to the OctoEverywhere service. + - Learn more here: https://octoeverywhere.com/s/bambu-setup +- Or if you're connecting in LAN Only Mode... - Your printer's Access Code - https://octoeverywhere.com/s/access-code -These three values must be set at environment vars when you first run the container. Once the container is ran, you don't need to include them, unless you want to update the values. - -- SERIAL_NUMBER=(serial number) -- PRINTER_IP=(ip address) -- If connecting via Bambu Cloud: - - BAMBU_CLOUD_ACCOUNT_EMAIL=(email) - - BAMBU_CLOUD_ACCOUNT_PASSWORD=(password) - - Optional - BAMBU_CLOUD_REGION=china - Use if your Bambu account is in the China region. -- If connecting via LAN Only Mode: - - ACCESS_CODE=(code) - - LAN_ONLY_MODE=TRUE - - -## Required Persistent Storage - -You must map the `/data` folder in your docker container to a directory on your computer so the plugin can write data that will remain between runs. Failure to do this will require relinking the plugin when the container is destroyed or updated. - ## Linking Your Bambu Connect Plugin -Once the docker container is running, you need to look at the logs to find the linking URL. +Once the docker container is running, you need to view the logs to find the linking URL. Docker Compose: `docker compose logs | grep https://octoeverywhere.com/getstarted` @@ -83,19 +63,31 @@ Using docker compose is the easiest way to run OctoEverywhere's Bambu Connect us - Install [Docker and Docker Compose](https://docs.docker.com/compose/install/linux/) - Clone this repo -- Edit the `./docker-compose.yml` file to enter your environment vars +- Edit the `./docker-compose.yml` file to enter your environment information.. - Run `docker compose up -d` - Follow the "Linking Your Bambu Connect Plugin" to link the plugin to your account. ## Using Docker -Docker compose is a fancy wrapper to run docker containers. You can also run docker containers manually. +These three values must be set at environment vars when you first run the container. Once the container is ran, you don't need to include them, unless you want to update the values. -Use a command like this example, but update the required vars. +- SERIAL_NUMBER=(serial number) +- PRINTER_IP=(ip address) +- If connecting via Bambu Cloud... + - BAMBU_CLOUD_ACCOUNT_EMAIL=(email) + - BAMBU_CLOUD_ACCOUNT_PASSWORD=(password) + - Optional - BAMBU_CLOUD_REGION=china - Use if your Bambu account is in the China region. +- If connecting via LAN Only Mode... + - ACCESS_CODE=(code) + - LAN_ONLY_MODE=TRUE + +Pull the docker container locally: `docker pull octoeverywhere/octoeverywhere` -`docker run --name bambu-connect -e SERIAL_NUMBER= -e PRINTER_IP= -e BAMBU_CLOUD_ACCOUNT_EMAIL="" -e BAMBU_CLOUD_ACCOUNT_PASSWORD="" -v /your/local/data/folder:/data -d octoeverywhere/octoeverywhere` +Run the docker container passing the required information: + +`docker run --name bambu-connect -e SERIAL_NUMBER= -e PRINTER_IP= -e BAMBU_CLOUD_ACCOUNT_EMAIL="" -e BAMBU_CLOUD_ACCOUNT_PASSWORD="" -v ./data:/data -d octoeverywhere/octoeverywhere` Follow the "Linking Your Bambu Connect Plugin" to link the plugin to your account. diff --git a/setup.py b/setup.py index 19932e0..53e5800 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this single version string is used by all of the plugins in OctoEverywhere! -plugin_version = "3.4.0" +plugin_version = "3.4.1" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From c21747f5b712cffcd2dacc294eef80cd0c6ef956 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Thu, 27 Jun 2024 08:12:46 -0700 Subject: [PATCH 113/328] Moving the crypto lib to a an optional lib in the installer. --- .github/workflows/pylint.yml | 2 ++ Dockerfile | 3 +++ py_installer/OptionalDepsInstaller.py | 24 ++++++++++++++++++++++-- requirements.txt | 2 +- setup.py | 2 +- 5 files changed, 29 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 85b2ce2..c35910c 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -20,12 +20,14 @@ jobs: - name: Install dependencies # We always install zstandard by hand, since it's an optional lib. # Ideally this version will stay in sync with Compression.ZStandardPipPackageString + # cryptography is only required for Bambu Connect. run: | python -m pip install --upgrade pip pip install pylint pip install octoprint pip install -r requirements.txt pip install "zstandard>=0.21.0,<0.23.0" + pip install cryptography>=42.0.8 - name: Analysing the code with pylint run: | diff --git a/Dockerfile b/Dockerfile index f32011c..12c9813 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,6 +36,9 @@ RUN ${VENV_DIR}/bin/pip3 install --require-virtualenv --no-cache-dir -q -r ${REP RUN apk add zstd RUN ${VENV_DIR}/bin/pip3 install --require-virtualenv --no-cache-dir -q "zstandard>=0.21.0,<0.23.0" +# Also install the crypto package used only for Bmabu Connect +RUN ${VENV_DIR}/bin/pip3 install --require-virtualenv --no-cache-dir -q "cryptography>=42.0.8" + # For docker, we use our docker_octoeverywhere host to handle the runtime setup and launch of the serivce. WORKDIR ${REPO_DIR} # Use the full path to the venv, we msut use this [] notation for our ctlc handler to work in the contianer diff --git a/py_installer/OptionalDepsInstaller.py b/py_installer/OptionalDepsInstaller.py index 516ab88..6c2c151 100644 --- a/py_installer/OptionalDepsInstaller.py +++ b/py_installer/OptionalDepsInstaller.py @@ -44,11 +44,31 @@ def WaitForInstallToComplete(timeoutSec:float=10.0) -> None: @staticmethod def _InstallThread(context:Context) -> None: + # Try to install zstandard, this is optional but recommended. + OptionalDepsInstaller._InstallZStandard(context) + # Try to install ffmpeg, this is required for RTSP streaming. OptionalDepsInstaller._DoFfmpegInstall(context) - # Try to install zstandard, this is optional but recommended. - OptionalDepsInstaller._InstallZStandard(context) + # This is only required for Bambu Connect. We can't put it in the requirements file because it doesn't work on the k1. + OptionalDepsInstaller._InstallCrypto(context) + + + @staticmethod + def _InstallCrypto(context:Context) -> None: + try: + # This is only required for Bambu Connect and will not install on the Creality OSes. + if context.OsType == OsTypes.K1 or context.OsType == OsTypes.SonicPad: + return + + # Now try to install the PY package. + # Only allow blocking up to 20 seconds, so we don't hang the installer too long. + startSec = time.time() + result = subprocess.run([sys.executable, '-m', 'pip', 'install', "cryptography>=42.0.8"], timeout=60.0, check=False, capture_output=True) + Logger.Debug(f"Cryptography PIP install result. Code: {result.returncode}, StdOut: {result.stdout}, StdErr: {result.stderr}, Time: {time.time()-startSec}") + + except Exception as e: + Logger.Debug(f"Error installing cryptography. {str(e)}") @staticmethod diff --git a/requirements.txt b/requirements.txt index e601644..dc4732a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,4 +24,4 @@ configparser # Only used for Bambu Connect paho-mqtt>=2.0.0 -cryptography>=42.0.8 \ No newline at end of file +#cryptography>=42.0.8 - Installed by the PY Installer / Dockerfile, since this breaks Creality printers. \ No newline at end of file diff --git a/setup.py b/setup.py index 53e5800..cdf9b6f 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this single version string is used by all of the plugins in OctoEverywhere! -plugin_version = "3.4.1" +plugin_version = "3.4.2" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 80b3074934c996927ce045a39c10c6bbd3651d0a Mon Sep 17 00:00:00 2001 From: Spencer Owen Date: Fri, 28 Jun 2024 01:12:44 -0600 Subject: [PATCH 114/328] Fix boolean (#68) --- docker_octoeverywhere/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker_octoeverywhere/__main__.py b/docker_octoeverywhere/__main__.py index ad0cf7f..956a9d8 100644 --- a/docker_octoeverywhere/__main__.py +++ b/docker_octoeverywhere/__main__.py @@ -99,7 +99,7 @@ def CreateDirIfNotExists(path: str) -> None: # The work around was to connect to the Bambu Cloud instead of directly to the printer. # The biggest downside of this is that we need to get the user's email address and password for Bambu Cloud. # BUT the user can also do the LAN only mode, if they want to. - isLanOnlyMode = os.environ.get("LAN_ONLY_MODE", False) + isLanOnlyMode = bool(os.environ.get("LAN_ONLY_MODE", "").lower() in ("true", "1", "yes")) isAccessCodeRequired = True if isLanOnlyMode: # In LAN only mode we only need the Serial number and access code. From 29246e0e6a69f137657891d0f8714296885fce4a Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Fri, 28 Jun 2024 00:10:44 -0700 Subject: [PATCH 115/328] Fixing the docker image. --- .github/workflows/docker-build-test.yml | 57 +++++++++++++++++++++++++ Dockerfile | 4 +- setup.py | 2 +- 3 files changed, 60 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/docker-build-test.yml diff --git a/.github/workflows/docker-build-test.yml b/.github/workflows/docker-build-test.yml new file mode 100644 index 0000000..662eedb --- /dev/null +++ b/.github/workflows/docker-build-test.yml @@ -0,0 +1,57 @@ +name: Publish Docker image + +# Triggers on our test branch for building docker. +on: + push: + branches: + - docker-build + +jobs: + push_to_registry: + name: Test Build Docker + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + # This is needed for the attestation step + id-token: write + attestations: write + steps: + - name: Check out the repo + uses: actions/checkout@v4 + + # Required for docker multi arch building. + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + # Required for docker multi arch building. + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: octoeverywhere/octoeverywhere + tags: | + # set latest tag + type=raw,value=latest + # set versioned tag + type=semver,pattern={{version}} + + - name: Build and push Docker image + id: push + uses: docker/build-push-action@v5 + with: + context: . + platforms: linux/amd64,linux/arm/v7,linux/arm64 + file: ./Dockerfile + push: false + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 12c9813..e43c26c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,8 +16,8 @@ ENV DATA_DIR=/data/ # Install the required packages. # Any packages here should be mirrored in the install script - and any optaionl pillow packages done inline. -# GCC, python3-dev, and musl-dev are required for pillow, and jpeg-dev and zlib-dev are required for jpeg support. -RUN apk add --no-cache curl ffmpeg jq python3 python3-dev gcc musl-dev py3-pip py3-virtualenv jpeg-dev libjpeg-turbo-dev zlib-dev py3-pillow +# GCC, python3-dev, and musl-dev are required for pillow, and jpeg-dev and zlib-dev are required for jpeg support. libffi-dev, rust, cargo, pkgconfig libressl-dev are requried to build the PY cryptography lib. +RUN apk add --no-cache curl ffmpeg jq python3 python3-dev gcc musl-dev py3-pip py3-virtualenv jpeg-dev libjpeg-turbo-dev zlib-dev py3-pillow libffi-dev rust cargo pkgconfig libressl-dev # # We decided to not run the installer, since the point of the installer is to setup the env, build the launch args, and setup the serivce. diff --git a/setup.py b/setup.py index cdf9b6f..4c18436 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this single version string is used by all of the plugins in OctoEverywhere! -plugin_version = "3.4.2" +plugin_version = "3.4.3" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From b085be1264e435aef8320fb0a6fb65622d0e5e9c Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Fri, 28 Jun 2024 17:39:37 -0700 Subject: [PATCH 116/328] Minor fixes to the update and uninstall scripts. --- uninstall.sh | 0 update.sh | 5 +++++ 2 files changed, 5 insertions(+) mode change 100644 => 100755 uninstall.sh diff --git a/uninstall.sh b/uninstall.sh old mode 100644 new mode 100755 diff --git a/update.sh b/update.sh index b6d00d5..6d4a35a 100755 --- a/update.sh +++ b/update.sh @@ -58,6 +58,11 @@ runAsRepoOwner() fi } +# Ensure we are cd'd into the repo dir. It's possible to run the update script outside of the repo dir. +# If it's ran from a different git repo, the git commands will try to effect it. +repoDir=$(readlink -f $(dirname "$0")) +cd $repoDir + # Pull the repo to get the top of master. echo "Updating repo and fetching the latest released tag..." runAsRepoOwner "git fetch --tags" From 26dce455b83cfeb592475a36c4ab6600f0ba6c18 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sat, 29 Jun 2024 08:55:28 -0700 Subject: [PATCH 117/328] Fixing some some installer logging and such. --- .vscode/settings.json | 2 + install.sh | 30 ++++++++------- octoeverywhere/telemetry.py | 12 +++--- py_installer/ConfigHelper.py | 2 +- py_installer/Configure.py | 2 +- py_installer/Context.py | 16 ++++---- py_installer/Discovery.py | 6 +-- py_installer/Frontend.py | 5 ++- py_installer/Installer.py | 5 ++- py_installer/OptionalDepsInstaller.py | 53 +++++++++++++++++++++------ py_installer/Permissions.py | 2 +- py_installer/Service.py | 24 ++++++------ py_installer/Updater.py | 2 +- py_installer/Util.py | 10 ++--- 14 files changed, 106 insertions(+), 65 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 1c57475..57cdf83 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -177,7 +177,9 @@ "pushall", "pushd", "Pylint", + "pypi", "pythoncompat", + "pythonhosted", "PYTHONPATH", "quickcam", "ratos", diff --git a/install.sh b/install.sh index 0deb56a..f3c4afe 100755 --- a/install.sh +++ b/install.sh @@ -245,34 +245,36 @@ install_or_update_system_dependencies() opkg install ${SONIC_PAD_DEP_LIST} || true pip3 install -q --no-cache-dir virtualenv else + # Print this before the date update, since it might prompt for the user's password. + log_info "Installing required system packages..." + log_important "You might be asked for your system password - this is required to install the required system packages." + # It seems a lot of printer control systems don't have the date and time set correctly, and then the fail # getting packages and other downstream things. We will will use our HTTP API to set the current UTC time. # Note that since cloudflare will auto force http -> https, we use https, but ignore cert errors, that could be # caused by an incorrect date. # Note some companion systems don't have curl installed, so this will fail. - log_info "Ensuring the system date and time is correct..." + #log_info "Ensuring the system date and time is correct..." sudo date -s `curl --insecure 'https://octoeverywhere.com/api/util/date' 2>/dev/null` || true # These we require to be installed in the OS. # Note we need to do this before we create our virtual environment - log_important "You might be asked for your system password - this is required to install the required system packages." - log_info "Installing required system packages..." - sudo apt update 1>/dev/null` 2>/dev/null` || true - sudo apt install --yes ${PKGLIST} + sudo apt update 1>/dev/null 2>/dev/null || true + sudo apt install --yes ${PKGLIST} 2>/dev/null # The PY lib Pillow depends on some system packages that change names depending on the OS. # The easiest way to do this was just to try to install them and ignore errors. # Most systems already have the packages installed, so this only fixes edge cases. # Notes on Pillow deps: https://pillow.readthedocs.io/en/latest/installation.html - log_info "Ensuring zlib is install for Pillow, it's ok if this package install fails." - sudo apt install --yes zlib1g-dev 2> /dev/null || true - sudo apt install --yes zlib-devel 2> /dev/null || true - sudo apt install --yes python-imaging 2> /dev/null || true - sudo apt install --yes python3-pil 2> /dev/null || true - sudo apt install --yes python3-pillow 2> /dev/null || true + #log_info "Ensuring zlib is install for Pillow, it's ok if this package install fails." + sudo apt install --yes zlib1g-dev 2>/dev/null || true + sudo apt install --yes zlib-devel 2>/dev/null || true + sudo apt install --yes python-imaging 2>/dev/null || true + sudo apt install --yes python3-pil 2>/dev/null || true + sudo apt install --yes python3-pillow 2>/dev/null || true fi - log_info "System package install complete." + #log_info "System package install complete." } # @@ -300,7 +302,7 @@ install_or_update_python_env() else "${OE_ENV}"/bin/pip3 install --require-virtualenv --no-cache-dir -q -r "${OE_REPO_DIR}"/requirements.txt fi - log_info "Python libs installed." + #log_info "Python libs installed." } # @@ -410,7 +412,7 @@ USERNAME=${USER} USER_HOME=${HOME} CMD_LINE_ARGS=${@} PY_LAUNCH_JSON="{\"OE_REPO_DIR\":\"${OE_REPO_DIR}\",\"OE_ENV\":\"${OE_ENV}\",\"USERNAME\":\"${USERNAME}\",\"USER_HOME\":\"${USER_HOME}\",\"CMD_LINE_ARGS\":\"${CMD_LINE_ARGS}\"}" -log_info "Bootstrap done. Starting python installer..." +#log_info "Bootstrap done. Starting python installer..." # Now launch into our py setup script, that does everything else required. # Since we use a module for file includes, we need to set the path to the root of the module diff --git a/octoeverywhere/telemetry.py b/octoeverywhere/telemetry.py index 1033f86..aef6957 100644 --- a/octoeverywhere/telemetry.py +++ b/octoeverywhere/telemetry.py @@ -1,4 +1,6 @@ +import logging import threading + import requests # A helper class for reporting telemetry. @@ -7,7 +9,7 @@ class Telemetry: ServerProtocolAndDomain = "https://octoeverywhere.com" @staticmethod - def Init(logger): + def Init(logger:logging.Logger): Telemetry.Logger = logger # Sends a telemetry data point to the service. These data points are suggestions, they are filtered and limited @@ -20,13 +22,13 @@ def Init(logger): # # Example: Telemetry.Write("Test", 1, { "FieldKey":"FieldValue", "FieldKey2":1.5 }, { "TagKey":"TagValue" }) @staticmethod - def Write(measureStr, valueInt, fieldsOpt, tagsOpt): - thread = threading.Thread(target=Telemetry.WriteSync, args=(measureStr, valueInt, fieldsOpt, tagsOpt, )) + def Write(measureStr:str, valueInt:int, fieldsOpt:dict=None, tagsOpt:dict=None): + thread = threading.Thread(target=Telemetry._WriteSync, args=(measureStr, valueInt, fieldsOpt, tagsOpt, )) thread.start() # Same as Write(), but it blocks on the request. True is returned on success, otherwise False. @staticmethod - def WriteSync(measureStr, valueInt, fieldsOpt, tagsOpt): + def _WriteSync(measureStr:str, valueInt:int, fieldsOpt:dict=None, tagsOpt:dict=None): try: # Ensure a value is set and ensure it's an int. if valueInt is None : @@ -61,5 +63,5 @@ def WriteSync(measureStr, valueInt, fieldsOpt, tagsOpt): @staticmethod - def SetServerProtocolAndDomain(protocolAndDomain): + def SetServerProtocolAndDomain(protocolAndDomain:str): Telemetry.ServerProtocolAndDomain = protocolAndDomain diff --git a/py_installer/ConfigHelper.py b/py_installer/ConfigHelper.py index 43d631b..f0bfd9f 100644 --- a/py_installer/ConfigHelper.py +++ b/py_installer/ConfigHelper.py @@ -164,7 +164,7 @@ def _GetConfig(context:Context = None, configFolderPath:str = None, createIfNotE if ConfigHelper.DoesConfigFileExist(context, configFolderPath) is False: if createIfNotExisting: # Fallthrough, the Config class will create a file if none exists. - Logger.Info("Creating main plugin config file.") + Logger.Debug("Creating main plugin config file.") else: return None # Get the config folder path. diff --git a/py_installer/Configure.py b/py_installer/Configure.py index 58cb3e2..2248838 100644 --- a/py_installer/Configure.py +++ b/py_installer/Configure.py @@ -137,4 +137,4 @@ def Run(self, context:Context): bc.EnsureBambuConnection(context) # Report - Logger.Info(f'Configured. Service: {context.ServiceName}, Path: {context.ServiceFilePath}, LocalStorage: {context.LocalFileStorageFolder}, Config Dir: {context.ConfigFolder}, Logs: {context.LogsFolder}') + Logger.Debug(f'Configured. Service: {context.ServiceName}, Path: {context.ServiceFilePath}, LocalStorage: {context.LocalFileStorageFolder}, Config Dir: {context.ConfigFolder}, Logs: {context.LogsFolder}') diff --git a/py_installer/Context.py b/py_installer/Context.py index ce7a5d8..40ee502 100644 --- a/py_installer/Context.py +++ b/py_installer/Context.py @@ -2,10 +2,11 @@ import json from enum import Enum +from octoeverywhere.telemetry import Telemetry + from .Logging import Logger from .Paths import Paths - # Indicates the OS type this installer is running on. class OsTypes(Enum): Debian = 1 @@ -239,23 +240,23 @@ def ParseCmdLineArgs(self): Logger.Warn("Skipping sudo actions. ! This will not result in a valid install! ") self.SkipSudoActions = True elif rawArgLower == "noatuoselect": - Logger.Info("Disabling Moonraker instance auto selection.") + Logger.Debug("Disabling Moonraker instance auto selection.") self.DisableAutoMoonrakerInstanceSelection = True elif rawArgLower == "observer": # This is the legacy flag - Logger.Info("Setup running in companion setup mode.") + Logger.Debug("Setup running in companion setup mode.") self.IsCompanionSetup = True elif rawArgLower == "companion": - Logger.Info("Setup running in companion setup mode.") + Logger.Debug("Setup running in companion setup mode.") self.IsCompanionSetup = True elif rawArgLower == "bambu": - Logger.Info("Setup running in Bambu Connect setup mode.") + Logger.Debug("Setup running in Bambu Connect setup mode.") self.IsBambuSetup = True elif rawArgLower == "update" or rawArgLower == "upgrade": - Logger.Info("Setup running in update mode.") + Logger.Debug("Setup running in update mode.") self.IsUpdateMode = True elif rawArgLower == "uninstall": - Logger.Info("Setup running in uninstall mode.") + Logger.Debug("Setup running in uninstall mode.") self.IsUninstallMode = True else: raise Exception("Unknown argument found. Use install.sh -help for options.") @@ -265,6 +266,7 @@ def ParseCmdLineArgs(self): if self.MoonrakerConfigFilePath is None: self.MoonrakerConfigFilePath = a Logger.Debug("Moonraker config file path found as argument:"+self.MoonrakerConfigFilePath) + Telemetry.Write("Installer-MoonrakerConfigPassed", 1) elif self.MoonrakerServiceFileName is None: self.MoonrakerServiceFileName = a Logger.Debug("Moonraker service file name found as argument:"+self.MoonrakerServiceFileName) diff --git a/py_installer/Discovery.py b/py_installer/Discovery.py index ae75054..f1d72e8 100644 --- a/py_installer/Discovery.py +++ b/py_installer/Discovery.py @@ -32,7 +32,7 @@ def FindTargetMoonrakerFiles(self, context:Context): if context.MoonrakerConfigFilePath is not None: if os.path.exists(context.MoonrakerConfigFilePath): if context.MoonrakerServiceFileName is not None and len(context.MoonrakerServiceFileName) > 0: - Logger.Info(f"Installer script was passed a valid Moonraker config and service name. [{context.MoonrakerServiceFileName}:{context.MoonrakerConfigFilePath}]") + Logger.Debug(f"Installer script was passed a valid Moonraker config and service name. [{context.MoonrakerServiceFileName}:{context.MoonrakerConfigFilePath}]") return # If we are here, we either have no service file name but a config path, or neither. @@ -58,7 +58,7 @@ def FindTargetMoonrakerFiles(self, context:Context): if p.MoonrakerConfigFilePath == context.MoonrakerConfigFilePath: # Update the context and return! context.MoonrakerServiceFileName = p.ServiceFileName - Logger.Info(f"The given moonraker config was found with a service file pair. [{context.MoonrakerServiceFileName}:{context.MoonrakerConfigFilePath}]") + Logger.Debug(f"The given moonraker config was found with a service file pair. [{context.MoonrakerServiceFileName}:{context.MoonrakerConfigFilePath}]") return Logger.Warn(f"Moonraker config path [{context.MoonrakerConfigFilePath}] was given, but no found pair matched it.") @@ -67,7 +67,7 @@ def FindTargetMoonrakerFiles(self, context:Context): # Update the context and return! context.MoonrakerConfigFilePath = pairList[0].MoonrakerConfigFilePath context.MoonrakerServiceFileName = pairList[0].ServiceFileName - Logger.Info(f"Only one moonraker instance was found, so we are using it! [{context.MoonrakerServiceFileName}:{context.MoonrakerConfigFilePath}]") + Logger.Debug(f"Only one moonraker instance was found, so we are using it! [{context.MoonrakerServiceFileName}:{context.MoonrakerConfigFilePath}]") return # If there are many found, as the user which they 0want to use. diff --git a/py_installer/Frontend.py b/py_installer/Frontend.py index ebaa2b8..b0f9ee8 100644 --- a/py_installer/Frontend.py +++ b/py_installer/Frontend.py @@ -36,7 +36,7 @@ def DoFrontendSetup(self, context:Context): Logger.Debug("Skipping frontend setup, there's no frontend for bambu connect.") return - Logger.Header("Starting Web Interface Setup") + Logger.Debug("Starting Web Interface Setup") # Try to get the existing configured port. (currentPort, frontendHint_CanBeNone) = ConfigHelper.TryToGetFrontendDetails(context) @@ -44,6 +44,7 @@ def DoFrontendSetup(self, context:Context): # There is already a config with a port setup. # Ask if the user wants to keep the current setup. Logger.Blank() + Logger.Blank() Logger.Info("A web interface is already setup:") msg = "" if frontendHint_CanBeNone is not None and frontendHint_CanBeNone.lower() != str(KnownFrontends.Unknown): @@ -109,7 +110,7 @@ def _GetDesiredFrontend(self, context:Context): except Exception as _: Logger.Warn("Invalid input, try again.") else: - Logger.Info("No web interfaces could be detected.") + Logger.Debug("No web interfaces could be detected.") # If we are here, the user selected m to do a manual frontend setup. diff --git a/py_installer/Installer.py b/py_installer/Installer.py index f5bdaad..1ee5272 100644 --- a/py_installer/Installer.py +++ b/py_installer/Installer.py @@ -1,6 +1,8 @@ import sys import traceback +from octoeverywhere.telemetry import Telemetry + from .Linker import Linker from .Logging import Logger from .Service import Service @@ -54,6 +56,7 @@ def _RunInternal(self): # As soon as we have the user home make the log file. Logger.InitFile(context.UserHomePath, context.UserName) + Telemetry.Init(Logger.GetPyLogger()) # Parse the original CmdLineArgs Logger.Debug("Parsing script cmd line args.") @@ -62,7 +65,7 @@ def _RunInternal(self): # Figure out the OS type we are installing on. # This can be a normal debian device, Sonic Pad, K1, or others. context.DetectOsType() - Logger.Info(f"Os Type Detected: {context.OsType}") + Logger.Debug(f"Os Type Detected: {context.OsType}") # Print this again now that the debug cmd flag is parsed, since it might be useful. if context.Debug: diff --git a/py_installer/OptionalDepsInstaller.py b/py_installer/OptionalDepsInstaller.py index 6c2c151..cb80155 100644 --- a/py_installer/OptionalDepsInstaller.py +++ b/py_installer/OptionalDepsInstaller.py @@ -15,35 +15,57 @@ class OptionalDepsInstaller: # If there's an installer thread, it will be stored here. - _InstallThread = None + _InstallThread:threading.Thread = None + _ThreadStatus:str = None + # Tries to install zstandard and ffmpeg, but this won't fail if the install fails. # The PIP install can take quite a long time (20-30 seconds) so we run in async. @staticmethod def TryToInstallDepsAsync(context:Context) -> None: + # Since might need to run a sudo command to apt-install, try to run it now so the user can enter their password + # Otherwise they will randomly be prompted in the middle of the setup. + # If this fails, it's no problem, we can still try to install the PIP packages. + Logger.Info("Installing system library dependencies. You might ask to enter your password...") + Util.RunShellCommand("sudo time", False) + # Since the pip and apt install can take a long time, do the install process async. - OptionalDepsInstaller._InstallThread = threading.Thread(target=OptionalDepsInstaller._InstallThread, args=(context,), daemon=True) + OptionalDepsInstaller._InstallThread = threading.Thread(target=OptionalDepsInstaller._InstallThreadWorker, args=(context,), daemon=True) OptionalDepsInstaller._InstallThread.start() @staticmethod - def WaitForInstallToComplete(timeoutSec:float=10.0) -> None: + def WaitForInstallToComplete(timeoutSec:float=30.0) -> None: # See if we started a thread. t = OptionalDepsInstaller._InstallThread if t is None: return - # If we did, report and try to join it. - # If this fails, it's no big deal, because the plugin runtime will also try to install zstandard. - Logger.Info("Finishing install... this might take a moment...") - try: - t.join(timeout=timeoutSec) - except Exception as e: - Logger.Debug(f"Failed to join optional installer thread. {str(e)}") + # Wait for everything to be done. + Logger.Info("Finishing system dependencies install. This might take a moment...") + start = time.time() + lastThreadStatus = None + while time.time() - start < timeoutSec: + # Sleep, then check if the thread is done. + time.sleep(2.0) + + # If there is a new status, report it, so the user knows things are happening + if OptionalDepsInstaller._ThreadStatus is not None and OptionalDepsInstaller._ThreadStatus != lastThreadStatus: + Logger.Info(OptionalDepsInstaller._ThreadStatus) + lastThreadStatus = OptionalDepsInstaller._ThreadStatus + + # Check if we are done, if so, return. + if t.is_alive() is False: + try: + t.join(5.0) + except Exception as e: + Logger.Debug(f"Failed to join optional installer thread. {str(e)}") + return + Logger.Debug("Timeout while waiting for system dependencies install to complete") @staticmethod - def _InstallThread(context:Context) -> None: + def _InstallThreadWorker(context:Context) -> None: # Try to install zstandard, this is optional but recommended. OptionalDepsInstaller._InstallZStandard(context) @@ -64,9 +86,10 @@ def _InstallCrypto(context:Context) -> None: # Now try to install the PY package. # Only allow blocking up to 20 seconds, so we don't hang the installer too long. startSec = time.time() + OptionalDepsInstaller._ThreadStatus = "Installing python crypto libs..." result = subprocess.run([sys.executable, '-m', 'pip', 'install', "cryptography>=42.0.8"], timeout=60.0, check=False, capture_output=True) Logger.Debug(f"Cryptography PIP install result. Code: {result.returncode}, StdOut: {result.stdout}, StdErr: {result.stderr}, Time: {time.time()-startSec}") - + OptionalDepsInstaller._ThreadStatus = "Python crypto libs install complete" except Exception as e: Logger.Debug(f"Error installing cryptography. {str(e)}") @@ -86,6 +109,7 @@ def _InstallZStandard(context:Context) -> None: # Try to install the system package, if possible. This might bring in a binary. # If this fails, the PY package might be able to still bring in a pre-built binary. Logger.Debug("Installing zstandard, this might take a moment...") + OptionalDepsInstaller._ThreadStatus = "Installing zstandard system libs..." startSec = time.time() (returnCode, stdOut, stdError) = Util.RunShellCommand("sudo apt-get install zstd -y", False) Logger.Debug(f"Zstandard apt install result. Code: {returnCode}, StdOut: {stdOut}, StdErr: {stdError}") @@ -93,8 +117,11 @@ def _InstallZStandard(context:Context) -> None: # Now try to install the PY package. # NOTE: Use the same logic as we do in the Compression class. # Only allow blocking up to 20 seconds, so we don't hang the installer too long. + OptionalDepsInstaller._ThreadStatus = "Installing zstandard python libs..." result = subprocess.run([sys.executable, '-m', 'pip', 'install', Compression.ZStandardPipPackageString], timeout=30.0, check=False, capture_output=True) Logger.Debug(f"Zstandard PIP install result. Code: {result.returncode}, StdOut: {result.stdout}, StdErr: {result.stderr}, Time: {time.time()-startSec}") + OptionalDepsInstaller._ZstandardInstallStatus = True + OptionalDepsInstaller._ThreadStatus = "Zstandard install complete" except Exception as e: Logger.Debug(f"Error installing zstandard. {str(e)}") @@ -109,9 +136,11 @@ def _DoFfmpegInstall(context:Context) -> None: # Try to install ffmpeg, this is required for RTSP streaming. Logger.Debug("Installing ffmpeg, this might take a moment...") + OptionalDepsInstaller._ThreadStatus = "Installing ffmpeg system libs..." startSec = time.time() (returnCode, stdOut, stdError) = Util.RunShellCommand("sudo apt-get install ffmpeg -y", False) # Report the status to the installer log. Logger.Debug(f"FFmpeg install result. Code: {returnCode}, StdOut: {stdOut}, StdErr: {stdError}, Time: {time.time()-startSec}") + OptionalDepsInstaller._ThreadStatus = "Ffmpeg install complete" except Exception as e: Logger.Debug(f"Error installing ffmpeg. {str(e)}") diff --git a/py_installer/Permissions.py b/py_installer/Permissions.py index e80ba93..1f24c5f 100644 --- a/py_installer/Permissions.py +++ b/py_installer/Permissions.py @@ -22,7 +22,7 @@ def CheckUserAndCorrectIfRequired_RanBeforeFirstContextValidation(self, context: if context.UserName is None or len(context.UserName) == 0: # Since the install script does a cd ~, we know if the user home path starts with /root/, the user is root. if context.UserHomePath is not None and context.UserHomePath.lower().startswith("/root/"): - Logger.Info("No user passed, but we detected the user is root.") + Logger.Debug("No user passed, but we detected the user is root.") context.UserName = Permissions.c_RootUserName diff --git a/py_installer/Service.py b/py_installer/Service.py index d032009..8643bc3 100644 --- a/py_installer/Service.py +++ b/py_installer/Service.py @@ -16,7 +16,7 @@ def Install(self, context:Context): # We always re-write the service file, to make sure it's current. if os.path.exists(context.ServiceFilePath): - Logger.Info("Service file already exists, recreating.") + Logger.Debug("Service file already exists, recreating.") # Create the service file. @@ -87,16 +87,16 @@ def _InstallDebian(self, context:Context, argsJsonBase64:str, moduleNameToRun:st return Logger.Debug("Service config file contents to write: "+s) - Logger.Info("Creating service file "+context.ServiceFilePath+"...") + Logger.Debug("Creating service file "+context.ServiceFilePath+"...") with open(context.ServiceFilePath, "w", encoding="utf-8") as serviceFile: serviceFile.write(s) - Logger.Info("Registering service...") + Logger.Debug("Registering service...") Util.RunShellCommand("systemctl enable "+context.ServiceName) Util.RunShellCommand("systemctl daemon-reload") # Stop and start to restart any running services. - Logger.Info("Starting service...") + Logger.Debug("Starting service...") Service.RestartDebianService(context.ServiceName) Logger.Info("Service setup and start complete!") @@ -133,15 +133,15 @@ def _InstallSonicPad(self, context:Context, argsJsonBase64:str, moduleNameToRun: return Logger.Debug("Service config file contents to write: "+s) - Logger.Info("Creating service file "+context.ServiceFilePath+"...") + Logger.Debug("Creating service file "+context.ServiceFilePath+"...") with open(context.ServiceFilePath, "w", encoding="utf-8") as serviceFile: serviceFile.write(s) # Make the script executable. - Logger.Info("Making the service executable...") + Logger.Debug("Making the service executable...") Util.RunShellCommand(f"chmod +x {context.ServiceFilePath}") - Logger.Info("Starting the service...") + Logger.Debug("Starting the service...") Service.RestartSonicPadService(context.ServiceFilePath) Logger.Info("Service setup and start complete!") @@ -211,26 +211,26 @@ def _InstallK1(self, context:Context, argsJsonBase64:str, moduleNameToRun:str): # Write the run script Logger.Debug("Run script file contents to write: "+runScriptContents) - Logger.Info("Creating service run script...") + Logger.Debug("Creating service run script...") with open(runScriptFilePath, "w", encoding="utf-8") as runScript: runScript.write(runScriptContents) # Make the script executable. - Logger.Info("Making the run script executable...") + Logger.Debug("Making the run script executable...") Util.RunShellCommand(f"chmod +x {runScriptFilePath}") # The file name is specific to the K1 and it's set in the Configure step. Logger.Debug("Service config file contents to write: "+serviceFileContents) - Logger.Info("Creating service file "+context.ServiceFilePath+"...") + Logger.Debug("Creating service file "+context.ServiceFilePath+"...") with open(context.ServiceFilePath, "w", encoding="utf-8") as serviceFile: serviceFile.write(serviceFileContents) # Make the script executable. - Logger.Info("Making the service executable...") + Logger.Debug("Making the service executable...") Util.RunShellCommand(f"chmod +x {context.ServiceFilePath}") # Use the common restart logic. - Logger.Info("Starting the service...") + Logger.Debug("Starting the service...") Service.RestartK1Service(context.ServiceFilePath) Logger.Info("Service setup and start complete!") diff --git a/py_installer/Updater.py b/py_installer/Updater.py index 909cd64..e925004 100644 --- a/py_installer/Updater.py +++ b/py_installer/Updater.py @@ -50,7 +50,7 @@ def DoUpdate(self, context:Context): # Before we restart the plugins, wait for the zstandard install to be done. # Give the updater extra time to work, since it's much shorter - OptionalDepsInstaller.WaitForInstallToComplete(timeoutSec=30.0) + OptionalDepsInstaller.WaitForInstallToComplete(timeoutSec=60.0) Logger.Info("We found the following plugins to update:") for s in foundOeServices: diff --git a/py_installer/Util.py b/py_installer/Util.py index 6a1d755..b6fdcab 100644 --- a/py_installer/Util.py +++ b/py_installer/Util.py @@ -29,21 +29,21 @@ def RunShellCommand(cmd:str, throwOnNonZeroReturnCode:bool = True): @staticmethod def EnsureDirExists(dirPath, context:Context, setPermissionsToUser = False): # Ensure it exists. - Logger.Header("Enuring path and permissions ["+dirPath+"]...") + Logger.Debug("Enuring path and permissions ["+dirPath+"]...") if os.path.exists(dirPath) is False: - Logger.Info("Dir doesn't exist, creating...") + Logger.Debug("Dir doesn't exist, creating...") os.mkdir(dirPath) else: - Logger.Info("Dir already exists.") + Logger.Debug("Dir already exists.") if setPermissionsToUser: - Logger.Info("Setting owner permissions to the service user ["+context.UserName+"]...") + Logger.Debug("Setting owner permissions to the service user ["+context.UserName+"]...") uid = pwd.getpwnam(context.UserName).pw_uid gid = pwd.getpwnam(context.UserName).pw_gid # pylint: disable=no-member # Linux only os.chown(dirPath, uid, gid) - Logger.Info("Directory setup successfully.") + Logger.Debug("Directory setup successfully.") # Ensures that all files and dirs down stream of this root dir path are owned by the requested user. From a672946e0041afbb1042d055383474c5a6e199b8 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sat, 29 Jun 2024 08:56:36 -0700 Subject: [PATCH 118/328] Adding back one logging line --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index f3c4afe..4b54ded 100755 --- a/install.sh +++ b/install.sh @@ -266,7 +266,7 @@ install_or_update_system_dependencies() # The easiest way to do this was just to try to install them and ignore errors. # Most systems already have the packages installed, so this only fixes edge cases. # Notes on Pillow deps: https://pillow.readthedocs.io/en/latest/installation.html - #log_info "Ensuring zlib is install for Pillow, it's ok if this package install fails." + log_info "Ensuring zlib is install for Pillow, it's ok if this package install fails." sudo apt install --yes zlib1g-dev 2>/dev/null || true sudo apt install --yes zlib-devel 2>/dev/null || true sudo apt install --yes python-imaging 2>/dev/null || true From 0aabdf6680fd8cea2be5affaf4f4865bbae221f9 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sat, 29 Jun 2024 14:41:18 -0700 Subject: [PATCH 119/328] Github action updates --- .github/workflows/pylint.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index c35910c..f58860e 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -11,9 +11,9 @@ jobs: # As of 4-23-2024 OctoPrint doesn't support 3.12, so we can't test it because it will fail the pip install. python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} From 4b425714fed6896248e5b5d36d286e6516a6a12f Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sat, 29 Jun 2024 15:47:12 -0700 Subject: [PATCH 120/328] Updating the link rules --- .github/workflows/pylint.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index f58860e..d0f7711 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -1,6 +1,8 @@ name: Pylint -on: [push] +on: + push: + pull_request: jobs: build: @@ -8,8 +10,7 @@ jobs: strategy: matrix: # The sonic pad runs 3.7, so it's important to keep it here to make sure all of our required dependencies work - # As of 4-23-2024 OctoPrint doesn't support 3.12, so we can't test it because it will fail the pip install. - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} From 9b8aacb0099656cb1b21f93bf11ab80ba0c1526a Mon Sep 17 00:00:00 2001 From: Spencer Owen Date: Sat, 29 Jun 2024 17:18:54 -0600 Subject: [PATCH 121/328] Add support for AP07 BambuLabs A1 Mini (#70) * Add support for AP07 a1mini, refactor if statements * revert lint * Lint --- bambu_octoeverywhere/bambumodels.py | 34 ++++++++++++++++------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/bambu_octoeverywhere/bambumodels.py b/bambu_octoeverywhere/bambumodels.py index db3a52f..e4fda12 100644 --- a/bambu_octoeverywhere/bambumodels.py +++ b/bambu_octoeverywhere/bambumodels.py @@ -232,21 +232,25 @@ def OnUpdate(self, msg:dict) -> None: # Now that we have info, map the printer type. if self.Cpu is not BambuCPUs.Unknown and self.HardwareVersion is not None: if self.Cpu is BambuCPUs.RV1126: - if self.HardwareVersion == "AP05": - self.PrinterName = BambuPrinters.X1C - elif self.HardwareVersion == "AP02": - self.PrinterName = BambuPrinters.X1E - if self.Cpu is BambuCPUs.ESP32 and self.ProjectName is not None: - if self.HardwareVersion == "AP04": - if self.ProjectName == "C11": - self.PrinterName = BambuPrinters.P1P - if self.ProjectName == "C12": - self.PrinterName = BambuPrinters.P1S - if self.HardwareVersion == "AP05": - if self.ProjectName == "N1": - self.PrinterName = BambuPrinters.A1Mini - if self.ProjectName == "N2S": - self.PrinterName = BambuPrinters.A1 + # Map for RV1126 CPU + rv1126_map = { + "AP05": BambuPrinters.X1C, + "AP02": BambuPrinters.X1E, + # Add more mappings here as needed + } + self.PrinterName = rv1126_map.get(self.HardwareVersion, BambuPrinters.Unknown) + + elif self.Cpu is BambuCPUs.ESP32 and self.ProjectName is not None: + # Map for ESP32 CPU + esp32_map = { + ("AP04", "C11"): BambuPrinters.P1P, + ("AP04", "C12"): BambuPrinters.P1S, + ("AP05", "N1"): BambuPrinters.A1Mini, + ("AP05", "N2S"): BambuPrinters.A1, + ("AP07", "N1"): BambuPrinters.A1Mini, + # Add more mappings here as needed + } + self.PrinterName = esp32_map.get((self.HardwareVersion, self.ProjectName), BambuPrinters.Unknown) if self.PrinterName is None or self.PrinterName is BambuPrinters.Unknown: Sentry.LogError(f"Unknown printer type. CPU:{self.Cpu}, Project Name: {self.ProjectName}, Hardware Version: {self.HardwareVersion}",{ From ef4a05ff2529f2c1543202cc2c6839b02cc5f7be Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sat, 29 Jun 2024 16:23:31 -0700 Subject: [PATCH 122/328] Tweaking the linting settings --- .pylintrc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pylintrc b/.pylintrc index e108966..434d551 100644 --- a/.pylintrc +++ b/.pylintrc @@ -81,7 +81,7 @@ persistent=yes # Min Python version to use for version dependend checks. Will default to the # version used to run pylint. -py-version=3.9 +# py-version=3.9 # When enabled, pylint would attempt to guess common misconfiguration and emit # user-friendly hints instead of false-positive error messages. @@ -124,7 +124,7 @@ disable=raw-checker-failed, # either give multiple identifier separated by comma (,) or put this option # multiple time (only on the command line, not in the configuration file where # it should appear only once). See also the "--disable" option for examples. -enable=c-extension-no-member +enable=c-extension-no-member,trailing-whitespace [REPORTS] From 4d42a5d2c8c54d7a5519406192ed773acce7c391 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Mon, 1 Jul 2024 06:34:56 -0700 Subject: [PATCH 123/328] Minor debugging edit. --- moonraker_octoeverywhere/moonrakerclient.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/moonraker_octoeverywhere/moonrakerclient.py b/moonraker_octoeverywhere/moonrakerclient.py index 318a497..e3b5baa 100644 --- a/moonraker_octoeverywhere/moonrakerclient.py +++ b/moonraker_octoeverywhere/moonrakerclient.py @@ -1058,7 +1058,9 @@ def GetCurrentLayerInfo(self): (objectHeight - firstLayerHeight) / layerHeight + 1) ) if totalLayers == 0: - self.Logger.error("GetCurrentLayerInfo failed to get a total layer count.") + if self.Logger.isEnabledFor(logging.DEBUG): + self.Logger.debug("GetCurrentLayerInfo failed to get a total layer count. "+json.dumps(printStats)) + self.Logger.warn("GetCurrentLayerInfo failed to get a total layer count.") return (0,0) # Next, try to get the current layer. From 8016a5cb306ad878c94c1dc2ed59c044959a438a Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Mon, 1 Jul 2024 06:43:02 -0700 Subject: [PATCH 124/328] Adding more debug logging. --- moonraker_octoeverywhere/moonrakerwebcamhelper.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/moonraker_octoeverywhere/moonrakerwebcamhelper.py b/moonraker_octoeverywhere/moonrakerwebcamhelper.py index fb0f378..b5e8ec4 100644 --- a/moonraker_octoeverywhere/moonrakerwebcamhelper.py +++ b/moonraker_octoeverywhere/moonrakerwebcamhelper.py @@ -137,8 +137,11 @@ def KickOffWebcamSettingsUpdate(self, forceUpdate = False): needToFindAutoSettings = len(self.AutoSettingsResults) == 0 if forceUpdate or needToFindAutoSettings or timeSinceLastWakeSec > MoonrakerWebcamHelper.c_MinTimeBetweenWebcamActivityInvokesSec: + self.Logger.info(f"Kicking the webcam setting read thread due to a request. Forced: {forceUpdate}, NeedAutoSettings: {needToFindAutoSettings}, TimeSinceLastWake: {timeSinceLastWakeSec} sec.") self.AutoSettingsLastWake = time.time() self.AutoSettingsWorkerEvent.set() + else: + self.Logger.debug(f"Ignoring the webcam setting read thread due to a request. Forced: {forceUpdate}, NeedAutoSettings: {needToFindAutoSettings}, TimeSinceLastWake: {timeSinceLastWakeSec} sec.") # This is the main worker thread that keeps track of webcam settings. From b501d0a9303f283b3fa9e753962a607472c9f59f Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Thu, 4 Jul 2024 11:34:33 -0700 Subject: [PATCH 125/328] Adding an os type telemetry message to help debug an issue from support. --- py_installer/Context.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/py_installer/Context.py b/py_installer/Context.py index 40ee502..39b29d3 100644 --- a/py_installer/Context.py +++ b/py_installer/Context.py @@ -8,6 +8,7 @@ from .Paths import Paths # Indicates the OS type this installer is running on. +# These can't changed, only added to, since they are using to write on disk and such. class OsTypes(Enum): Debian = 1 SonicPad = 2 @@ -285,6 +286,12 @@ def _ValidateString(self, s:str, error:str): def DetectOsType(self): + self._DetectOsType() + # TODO - Remove. Send telemetry about the OS type while we try to debug an issue on an unknown OS. + Telemetry.Write("Installer-OsType", 1, fieldsOpt={ "OsType": self.OsType }) + + + def _DetectOsType(self): # # Note! This should closely resemble the ostype.py class in the plugin and the logic in the ./install.sh script! # From ed4ac2147e426c0007ed6f74b3025d086797d82d Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sat, 6 Jul 2024 11:23:37 -0700 Subject: [PATCH 126/328] Fixing the OS Type Installer Debug Log --- py_installer/Context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py_installer/Context.py b/py_installer/Context.py index 39b29d3..208f205 100644 --- a/py_installer/Context.py +++ b/py_installer/Context.py @@ -288,7 +288,7 @@ def _ValidateString(self, s:str, error:str): def DetectOsType(self): self._DetectOsType() # TODO - Remove. Send telemetry about the OS type while we try to debug an issue on an unknown OS. - Telemetry.Write("Installer-OsType", 1, fieldsOpt={ "OsType": self.OsType }) + Telemetry.Write("Installer-OsType", int(self.OsType)) def _DetectOsType(self): From 75dd9d3bb8751482ba5d0e17d792d646fbee5b87 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sat, 6 Jul 2024 14:32:27 -0500 Subject: [PATCH 127/328] Fixing the enum to allow it be cast to an int --- py_installer/Context.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/py_installer/Context.py b/py_installer/Context.py index 208f205..770846e 100644 --- a/py_installer/Context.py +++ b/py_installer/Context.py @@ -1,6 +1,6 @@ import os import json -from enum import Enum +from enum import IntEnum from octoeverywhere.telemetry import Telemetry @@ -9,7 +9,7 @@ # Indicates the OS type this installer is running on. # These can't changed, only added to, since they are using to write on disk and such. -class OsTypes(Enum): +class OsTypes(IntEnum): Debian = 1 SonicPad = 2 K1 = 3 # Both the K1 and K1 Max From c192e26dcd3b12302cfc497b7a4de49ef5154beb Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Mon, 8 Jul 2024 21:11:00 -0700 Subject: [PATCH 128/328] Removing the depdency on the crypo lib, since it causes install issues. --- .github/workflows/pylint.yml | 2 -- .vscode/settings.json | 1 + Dockerfile | 7 ++----- bambu_octoeverywhere/bambucloud.py | 28 +++++++++++++++++++++------ docker-readme.md | 5 +++-- py_installer/OptionalDepsInstaller.py | 21 -------------------- requirements.txt | 3 +-- 7 files changed, 29 insertions(+), 38 deletions(-) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index d0f7711..3f28415 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -21,14 +21,12 @@ jobs: - name: Install dependencies # We always install zstandard by hand, since it's an optional lib. # Ideally this version will stay in sync with Compression.ZStandardPipPackageString - # cryptography is only required for Bambu Connect. run: | python -m pip install --upgrade pip pip install pylint pip install octoprint pip install -r requirements.txt pip install "zstandard>=0.21.0,<0.23.0" - pip install cryptography>=42.0.8 - name: Analysing the code with pylint run: | diff --git a/.vscode/settings.json b/.vscode/settings.json index 57cdf83..3ca2f05 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -226,6 +226,7 @@ "uipopupinvoker", "unauth", "unauthed", + "Unobfuscate", "updatemanager", "urandom", "userdata", diff --git a/Dockerfile b/Dockerfile index e43c26c..f32011c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,8 +16,8 @@ ENV DATA_DIR=/data/ # Install the required packages. # Any packages here should be mirrored in the install script - and any optaionl pillow packages done inline. -# GCC, python3-dev, and musl-dev are required for pillow, and jpeg-dev and zlib-dev are required for jpeg support. libffi-dev, rust, cargo, pkgconfig libressl-dev are requried to build the PY cryptography lib. -RUN apk add --no-cache curl ffmpeg jq python3 python3-dev gcc musl-dev py3-pip py3-virtualenv jpeg-dev libjpeg-turbo-dev zlib-dev py3-pillow libffi-dev rust cargo pkgconfig libressl-dev +# GCC, python3-dev, and musl-dev are required for pillow, and jpeg-dev and zlib-dev are required for jpeg support. +RUN apk add --no-cache curl ffmpeg jq python3 python3-dev gcc musl-dev py3-pip py3-virtualenv jpeg-dev libjpeg-turbo-dev zlib-dev py3-pillow # # We decided to not run the installer, since the point of the installer is to setup the env, build the launch args, and setup the serivce. @@ -36,9 +36,6 @@ RUN ${VENV_DIR}/bin/pip3 install --require-virtualenv --no-cache-dir -q -r ${REP RUN apk add zstd RUN ${VENV_DIR}/bin/pip3 install --require-virtualenv --no-cache-dir -q "zstandard>=0.21.0,<0.23.0" -# Also install the crypto package used only for Bmabu Connect -RUN ${VENV_DIR}/bin/pip3 install --require-virtualenv --no-cache-dir -q "cryptography>=42.0.8" - # For docker, we use our docker_octoeverywhere host to handle the runtime setup and launch of the serivce. WORKDIR ${REPO_DIR} # Use the full path to the venv, we msut use this [] notation for our ctlc handler to work in the contianer diff --git a/bambu_octoeverywhere/bambucloud.py b/bambu_octoeverywhere/bambucloud.py index 98939c4..fac6356 100644 --- a/bambu_octoeverywhere/bambucloud.py +++ b/bambu_octoeverywhere/bambucloud.py @@ -1,11 +1,11 @@ import json +import codecs import base64 import logging import threading from enum import Enum import requests -from cryptography.fernet import Fernet from linux_host.config import Config @@ -248,9 +248,11 @@ def SetContext(self, email:str, p:str) -> bool: # So at least it's not just plain text. data = {"email":email, "p":p} j = json.dumps(data) - f = Fernet(b"iyqYOs9QPwO5J6jW30uPJIxywhf7yLrvaRXLp5gi9OA=") - token = f.encrypt(j.encode()) - self.Config.SetStr(Config.SectionBambu, Config.BambuCloudContext, token.decode()) + # In the past we used the crypo lib to actually do crypto with a static key here in the code. + # But the crypo lib had a lot of native lib requirements and it caused install issues. + # Since we were using a static key anyways, we will just do this custom obfuscation function. + token = self._ObfuscateString(j) + self.Config.SetStr(Config.SectionBambu, Config.BambuCloudContext, token) return True except Exception as e: Sentry.Exception("Bambu Cloud set email exception", e) @@ -272,8 +274,7 @@ def GetContext(self, expectContextToExist = True): if expectContextToExist: self.Logger.error("No Bambu Cloud context found in the config file.") return (None, None) - f = Fernet(b"iyqYOs9QPwO5J6jW30uPJIxywhf7yLrvaRXLp5gi9OA=") - jsonStr = f.decrypt(token.encode()) + jsonStr = self._UnobfuscateString(token) data = json.loads(jsonStr) e = data.get("email", None) p = data.get("p", None) @@ -284,3 +285,18 @@ def GetContext(self, expectContextToExist = True): except Exception as e: Sentry.Exception("Bambu Cloud login exception", e) return (None, None) + + + # The goal here is just to obfuscate the string with a unique algo, so the email and password aren't just plain text in the config file. + def _ObfuscateString(self, s:str) -> str: + # First, base64 encode the string. + base64Str = base64.b64encode(s.encode(encoding="utf-8")) + # First, next, rotate it. + return codecs.encode(base64Str.decode(encoding="utf-8"), 'rot13') + + + def _UnobfuscateString(self, s:str) -> str: + # Un-rotate + base64String = codecs.decode(s, 'rot13') + # Un-base64 encode + return base64.b64decode(base64String).decode(encoding="utf-8") diff --git a/docker-readme.md b/docker-readme.md index f9769df..eac90b4 100644 --- a/docker-readme.md +++ b/docker-readme.md @@ -20,7 +20,7 @@ Note if you put your printer in "LAN Only Mode" you **can** still use Bambu Stud For OctoEverywhere to connect to your 3D printer through Bambu Cloud, you just need to supply your Bambu Cloud account info to the local plugin. -**Rest assured, your Bambu Cloud email address and password are stored locally, encrypted on disk, and are never sent to the OctoEverywhere service.** +**Rest assured, your Bambu Cloud email address and password are stored locally, secured on disk, and are never sent to the OctoEverywhere service.** If you use Facebook, Google, or Apple to login to Bambu Cloud, [follow this guide to set a password on your account.](https://intercom.help/octoeverywhere/en/articles/9529936-bambu-cloud-with-bambu-connect) @@ -40,7 +40,7 @@ To use the Bambu Connect plugin, you need to get the following information. - If you're connecting with Bambu Cloud... - Your Bambu Cloud account email address - Your Bambu Cloud account password - - **Note:** Your Bambu Cloud email address and password are stored locally, encrypted on disk, and never sent to the OctoEverywhere service. + - **Note:** Your Bambu Cloud email address and password are stored locally, secured on disk, and never sent to the OctoEverywhere service. - Learn more here: https://octoeverywhere.com/s/bambu-setup - Or if you're connecting in LAN Only Mode... - Your printer's Access Code - https://octoeverywhere.com/s/access-code @@ -88,6 +88,7 @@ Pull the docker container locally: Run the docker container passing the required information: `docker run --name bambu-connect -e SERIAL_NUMBER= -e PRINTER_IP= -e BAMBU_CLOUD_ACCOUNT_EMAIL="" -e BAMBU_CLOUD_ACCOUNT_PASSWORD="" -v ./data:/data -d octoeverywhere/octoeverywhere` +`docker run --name bambu-connect -e SERIAL_NUMBER=test -e PRINTER_IP=1.1.1.1 -e LAN_ONLY_MODE=1 -v /data:/data -d octoeverywhere-local` Follow the "Linking Your Bambu Connect Plugin" to link the plugin to your account. diff --git a/py_installer/OptionalDepsInstaller.py b/py_installer/OptionalDepsInstaller.py index cb80155..5be66f2 100644 --- a/py_installer/OptionalDepsInstaller.py +++ b/py_installer/OptionalDepsInstaller.py @@ -72,27 +72,6 @@ def _InstallThreadWorker(context:Context) -> None: # Try to install ffmpeg, this is required for RTSP streaming. OptionalDepsInstaller._DoFfmpegInstall(context) - # This is only required for Bambu Connect. We can't put it in the requirements file because it doesn't work on the k1. - OptionalDepsInstaller._InstallCrypto(context) - - - @staticmethod - def _InstallCrypto(context:Context) -> None: - try: - # This is only required for Bambu Connect and will not install on the Creality OSes. - if context.OsType == OsTypes.K1 or context.OsType == OsTypes.SonicPad: - return - - # Now try to install the PY package. - # Only allow blocking up to 20 seconds, so we don't hang the installer too long. - startSec = time.time() - OptionalDepsInstaller._ThreadStatus = "Installing python crypto libs..." - result = subprocess.run([sys.executable, '-m', 'pip', 'install', "cryptography>=42.0.8"], timeout=60.0, check=False, capture_output=True) - Logger.Debug(f"Cryptography PIP install result. Code: {result.returncode}, StdOut: {result.stdout}, StdErr: {result.stderr}, Time: {time.time()-startSec}") - OptionalDepsInstaller._ThreadStatus = "Python crypto libs install complete" - except Exception as e: - Logger.Debug(f"Error installing cryptography. {str(e)}") - @staticmethod def _InstallZStandard(context:Context) -> None: diff --git a/requirements.txt b/requirements.txt index dc4732a..d02997c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,5 +23,4 @@ sentry-sdk>=1.19.1,<2 configparser # Only used for Bambu Connect -paho-mqtt>=2.0.0 -#cryptography>=42.0.8 - Installed by the PY Installer / Dockerfile, since this breaks Creality printers. \ No newline at end of file +paho-mqtt>=2.0.0 \ No newline at end of file From 19524166cdd25e69a25e78f5d6ac85eae60c6e29 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Mon, 8 Jul 2024 21:55:02 -0700 Subject: [PATCH 129/328] Version bump --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4c18436..378a3ee 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this single version string is used by all of the plugins in OctoEverywhere! -plugin_version = "3.4.3" +plugin_version = "3.4.4" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 3d9068567cce62aa914f06e85b0d0ee506bf5553 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Tue, 9 Jul 2024 14:31:47 -0700 Subject: [PATCH 130/328] Very small bug fix. --- octoeverywhere/octohttprequest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octoeverywhere/octohttprequest.py b/octoeverywhere/octohttprequest.py index deb1633..3b10f6c 100644 --- a/octoeverywhere/octohttprequest.py +++ b/octoeverywhere/octohttprequest.py @@ -449,7 +449,7 @@ def MakeHttpCallAttempt(logger, attemptName, method, url, headers, data, mainRes # use capture the main result object, so we can use it eventually if all fallbacks fail. result = None if response is not None: - OctoHttpRequest.Result(response.status_code, response.headers, url, isFallback, requestLibResponseObj=response) + result = OctoHttpRequest.Result(response.status_code, response.headers, url, isFallback, requestLibResponseObj=response) return OctoHttpRequest.AttemptResult(False, result) # We don't have another fallback, so we need to end this. From 4bb40bf4eb8a863abc3ae3bbe4fbfbd89257c4c1 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sat, 13 Jul 2024 14:22:33 -0700 Subject: [PATCH 131/328] Small change to fix the docker build. --- Dockerfile | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index f32011c..aae9e4d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ ENV DATA_DIR=/data/ # Install the required packages. # Any packages here should be mirrored in the install script - and any optaionl pillow packages done inline. # GCC, python3-dev, and musl-dev are required for pillow, and jpeg-dev and zlib-dev are required for jpeg support. -RUN apk add --no-cache curl ffmpeg jq python3 python3-dev gcc musl-dev py3-pip py3-virtualenv jpeg-dev libjpeg-turbo-dev zlib-dev py3-pillow +RUN apk add --no-cache curl ffmpeg jq python3 python3-dev gcc musl-dev py3-pip py3-virtualenv jpeg-dev libjpeg-turbo-dev zlib-dev py3-pillow libffi-dev # # We decided to not run the installer, since the point of the installer is to setup the env, build the launch args, and setup the serivce. diff --git a/setup.py b/setup.py index 378a3ee..1699616 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this single version string is used by all of the plugins in OctoEverywhere! -plugin_version = "3.4.4" +plugin_version = "3.4.5" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From d7b70221dd6f8fc281c353e4a9a1908258aa9fe9 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Tue, 30 Jul 2024 22:21:56 -0700 Subject: [PATCH 132/328] Adding logic provide a device ID in the connection handshake to help dedup plugins with backup/restored config files. --- .vscode/settings.json | 8 ++ bambu_octoeverywhere/bambuhost.py | 4 + moonraker_octoeverywhere/moonrakerhost.py | 4 + octoeverywhere/Proto/HandshakeSyn.py | 15 ++- octoeverywhere/deviceid.py | 127 ++++++++++++++++++++++ octoeverywhere/hostcommon.py | 6 +- octoeverywhere/notificationshandler.py | 4 +- octoeverywhere/octosessionimpl.py | 7 +- octoeverywhere/octostreammsgbuilder.py | 7 +- octoeverywhere/serverauth.py | 4 +- octoprint_octoeverywhere/__init__.py | 4 + octoprint_octoeverywhere/__main__.py | 8 +- octoprint_octoeverywhere/localauth.py | 4 +- setup.py | 2 +- 14 files changed, 189 insertions(+), 15 deletions(-) create mode 100644 octoeverywhere/deviceid.py diff --git a/.vscode/settings.json b/.vscode/settings.json index 3ca2f05..d7f0818 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -42,14 +42,17 @@ "crowsnest", "crypo", "Damerell", + "dbus", "decompressor", "decompressors", "deps", "devel", + "deviceid", "devs", "DGRAM", "didnt", "dnspython", + "dnstest", "esac", "faststart", "Fernet", @@ -76,12 +79,14 @@ "hacky", "handshakesyn", "hostcommon", + "hostid", "hostnames", "Hotend", "httpx", "INET", "inited", "ints", + "ioreg", "ipcam", "JFIF", "jmpeg", @@ -90,6 +95,7 @@ "jsonify", "kbytes", "keepalive", + "kenv", "keyvalidator", "KIAUH", "Klipper", @@ -115,9 +121,11 @@ "moonrakerdatabase", "moonrakerhost", "moonrakerwebcamhelper", + "mountinfo", "movflags", "mqtt", "msgcount", + "msys", "multicam", "myprinter", "nbsp", diff --git a/bambu_octoeverywhere/bambuhost.py b/bambu_octoeverywhere/bambuhost.py index 955d601..eef5d1c 100644 --- a/bambu_octoeverywhere/bambuhost.py +++ b/bambu_octoeverywhere/bambuhost.py @@ -3,6 +3,7 @@ from octoeverywhere.mdns import MDns from octoeverywhere.sentry import Sentry +from octoeverywhere.deviceid import DeviceId from octoeverywhere.telemetry import Telemetry from octoeverywhere.hostcommon import HostCommon from octoeverywhere.compression import Compression @@ -103,6 +104,9 @@ def RunBlocking(self, configPath, localStorageDir, repoRoot, devConfig_CanBeNone # Init the mdns client MDns.Init(self.Logger, localStorageDir) + # Init device id + DeviceId.Init(self.Logger) + # Setup the print info manager. PrintInfoManager.Init(self.Logger, localStorageDir) diff --git a/moonraker_octoeverywhere/moonrakerhost.py b/moonraker_octoeverywhere/moonrakerhost.py index 7fc922d..7e8e91e 100644 --- a/moonraker_octoeverywhere/moonrakerhost.py +++ b/moonraker_octoeverywhere/moonrakerhost.py @@ -3,6 +3,7 @@ from octoeverywhere.mdns import MDns from octoeverywhere.sentry import Sentry +from octoeverywhere.deviceid import DeviceId from octoeverywhere.telemetry import Telemetry from octoeverywhere.hostcommon import HostCommon from octoeverywhere.compression import Compression @@ -127,6 +128,9 @@ def RunBlocking(self, klipperConfigDir, isCompanionMode, localStorageDir, servic # Init the mdns client MDns.Init(self.Logger, localStorageDir) + # Init device id + DeviceId.Init(self.Logger) + # Allow the UI injector to run and do it's thing. UiInjector.Init(self.Logger, repoRoot) diff --git a/octoeverywhere/Proto/HandshakeSyn.py b/octoeverywhere/Proto/HandshakeSyn.py index ddf0c56..5b7ad1f 100644 --- a/octoeverywhere/Proto/HandshakeSyn.py +++ b/octoeverywhere/Proto/HandshakeSyn.py @@ -169,8 +169,15 @@ def ReceiveCompressionType(self): return self._tab.Get(octoflatbuffers.number_types.Int8Flags, o + self._tab.Pos) return 2 + # HandshakeSyn + def DeviceId(self) -> Optional[str]: + o = octoflatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(38)) + if o != 0: + return self._tab.String(o + self._tab.Pos) + return None + def HandshakeSynStart(builder: octoflatbuffers.Builder): - builder.StartObject(17) + builder.StartObject(18) def Start(builder: octoflatbuffers.Builder): HandshakeSynStart(builder) @@ -283,6 +290,12 @@ def HandshakeSynAddReceiveCompressionType(builder: octoflatbuffers.Builder, rece def AddReceiveCompressionType(builder: octoflatbuffers.Builder, receiveCompressionType: int): HandshakeSynAddReceiveCompressionType(builder, receiveCompressionType) +def HandshakeSynAddDeviceId(builder: octoflatbuffers.Builder, deviceId: int): + builder.PrependUOffsetTRelativeSlot(17, octoflatbuffers.number_types.UOffsetTFlags.py_type(deviceId), 0) + +def AddDeviceId(builder: octoflatbuffers.Builder, deviceId: int): + HandshakeSynAddDeviceId(builder, deviceId) + def HandshakeSynEnd(builder: octoflatbuffers.Builder) -> int: return builder.EndObject() diff --git a/octoeverywhere/deviceid.py b/octoeverywhere/deviceid.py new file mode 100644 index 0000000..930777a --- /dev/null +++ b/octoeverywhere/deviceid.py @@ -0,0 +1,127 @@ +import re +import sys +import logging +import platform +import subprocess + +from .sentry import Sentry + +# A class that tries to get a unique id per device that doesn't change, ideally even when the OS is re-installed. +# Inspired by https://github.com/keygen-sh/py-machineid/blob/master/machineid/__init__.py +class DeviceId: + + _Instance = None + + @staticmethod + def Init(logger: logging.Logger): + DeviceId._Instance = DeviceId(logger) + + + @staticmethod + def Get(): + return DeviceId._Instance + + + def __init__(self, logger: logging.Logger) -> None: + self.Logger = logger + + + # Get's a unique ID for the platform. The ID should be unique per platform and ideally not change even when the OS is re-installed. + # This ID can't not be written to disk it must come from the system level some how. + # If nothing can be found, None is return. + def GetId(self) -> str: + try: + return self._GetIdInternal() + except Exception as e: + Sentry.Exception("Exception in DeviceId.GetId", e) + return None + + + def _GetIdInternal(self) -> str: + # We have a few options to get a unique id for the device. + # Try each possible method and return the first one that works. + # We prefix each system, to ensure there are no collisions + + # Mac + if sys.platform == "darwin": + fid = self._RunCmd("ioreg -d2 -c IOPlatformExpertDevice | awk -F\\\" '/IOPlatformUUID/{print $(NF-1)}'") + if fid is not None: + self.Logger.debug(f"Found device id from darwin device id: {fid}") + return self._BuildId("darwin", fid) + + # Windows + if sys.platform in ('win32', 'cygwin', 'msys'): + self.Logger.debug("Windows is not supported in DeviceId right now.") + return None + + # Linux + if sys.platform.startswith("linux"): + fid = self._ReadFile("/var/lib/dbus/machine-id") + if fid is not None: + self.Logger.debug(f"Found device id from /var/lib/dbus/machine-id: {fid}") + return self._BuildId("linux-mi", fid) + + fid = self._ReadFile('/etc/machine-id') + if fid is not None: + self.Logger.debug(f"Found device id from /etc/machine-id: {fid}") + return self._BuildId("linux-mie", fid) + + group = self._ReadFile('/proc/self/cgroup') + if group is not None and 'docker' in group: + fid = self._RunCmd("head -1 /proc/self/cgroup | cut -d/ -f3") + if fid is not None: + self.Logger.debug(f"Found device id from docker cgroup: {fid}") + return self._BuildId("linux-d", fid) + + mountInfo = self._ReadFile('/proc/self/mountinfo') + if mountInfo and 'docker' in mountInfo: + fid = self._RunCmd("grep -oP '(?<=docker/containers/)([a-f0-9]+)(?=/hostname)' /proc/self/mountinfo") + if fid is not None: + self.Logger.debug(f"Found device id from docker mountinfo: {fid}") + return self._BuildId("linux-dm", fid) + + if 'microsoft' in platform.uname().release: + fid = self._RunCmd("powershell.exe -ExecutionPolicy bypass -command '(Get-CimInstance -Class Win32_ComputerSystemProduct).UUID'") + if fid is not None: + self.Logger.debug(f"Found device id from wsl UUID: {fid}") + return self._BuildId("wsl", fid) + + # BSD + if sys.platform.startswith(('openbsd', 'freebsd')): + fid = self._ReadFile("/etc/hostid") + if fid is not None: + self.Logger.debug(f"Found device id from /etc/hostid: {fid}") + return self._BuildId("bsd-h", fid) + + fid = self._RunCmd('kenv -q smbios.system.uuid') + if fid is not None: + self.Logger.debug(f"Found device id from kenv -q smbios.system.uuid: {fid}") + return self._BuildId("bsd-k", fid) + + self.Logger.warn(f"Found device ID not found on platform: {sys.platform}") + return None + + + # If the file exists and is readable, returns the body. + # Otherwise None + def _ReadFile(self, path: str) -> str: + try: + with open(path, encoding="utf-8") as f: + return f.read().strip() + except Exception: + return None + + + # Runs the command and returns stdout. + # Otherwise None + def _RunCmd(self, cmd: str) -> str: + try: + return subprocess.run(cmd, shell=True, capture_output=True, check=True, encoding="utf-8").stdout.strip() + except Exception: + return None + + + # Normalize the id to remove any whitespace or control characters. + # We add a prefix for each method to ensure they don't collide. + def _BuildId(self, method:str, fid: str) -> str: + return method + "-" + re.sub(r'[\x00-\x1f\x7f-\x9f\s]', '', fid).strip() diff --git a/octoeverywhere/hostcommon.py b/octoeverywhere/hostcommon.py index 27f90ef..f1edd1e 100644 --- a/octoeverywhere/hostcommon.py +++ b/octoeverywhere/hostcommon.py @@ -1,5 +1,5 @@ -import random import string +import secrets # Common functions that the hosts might need to use. class HostCommon: @@ -25,12 +25,12 @@ class HostCommon: # Returns a new printer Id. This needs to be crypo-random to make sure it's not predictable. @staticmethod def GeneratePrinterId(): - return ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(HostCommon.c_OctoEverywherePrinterIdMaxLength)) + return ''.join(secrets.choice(string.ascii_uppercase + string.digits) for _ in range(HostCommon.c_OctoEverywherePrinterIdMaxLength)) # Returns a new private key. This needs to be crypo-random to make sure it's not predictable. @staticmethod def GeneratePrivateKey(): - return ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(HostCommon.c_OctoEverywherePrivateKeyMinLength)) + return ''.join(secrets.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(HostCommon.c_OctoEverywherePrivateKeyMinLength)) @staticmethod def IsPrinterIdValid(printerId): diff --git a/octoeverywhere/notificationshandler.py b/octoeverywhere/notificationshandler.py index 0b92668..c451980 100644 --- a/octoeverywhere/notificationshandler.py +++ b/octoeverywhere/notificationshandler.py @@ -2,7 +2,7 @@ import time import io import threading -import random +import secrets import string import logging @@ -142,7 +142,7 @@ def _RecoverOrRestForNewPrint(self, printCookie:str): # Each time a print starts, we generate a fixed length random id to identify it. # This id is used to globally identify the print for the user, so it needs to have high entropy. - printId = ''.join(random.choices(string.ascii_uppercase + string.ascii_lowercase + string.digits, k=NotificationsHandler.PrintIdLength)) + printId = ''.join(secrets.choice(string.ascii_uppercase + string.digits) for _ in range(NotificationsHandler.PrintIdLength)) # Always make a new print info for this new print. # This is where we will store all of the vars for this print, and it's also written to disk if we need to recover the info. diff --git a/octoeverywhere/octosessionimpl.py b/octoeverywhere/octosessionimpl.py index 5db8880..5a1cfdc 100644 --- a/octoeverywhere/octosessionimpl.py +++ b/octoeverywhere/octosessionimpl.py @@ -18,6 +18,7 @@ from .ostypeidentifier import OsTypeIdentifier from .threaddebug import ThreadDebug from .compression import Compression +from .deviceid import DeviceId from .Proto import OctoStreamMessage from .Proto import HandshakeAck @@ -251,10 +252,14 @@ def StartHandshake(self, summonMethod): if Compression.Get().CanUseZStandardLib: receiveCompressionType = DataCompression.ZStandard + # If possible, get a device ID for this plugin. + # This will return None if no device id can be found. + deviceId = DeviceId.Get().GetId() + # Build the message buf = OctoStreamMsgBuilder.BuildHandshakeSyn(self.PrinterId, self.PrivateKey, self.isPrimarySession, self.PluginVersion, OctoHttpRequest.GetLocalHttpProxyPort(), LocalIpHelper.TryToGetLocalIp(), - rasChallenge, rasChallengeKeyVerInt, summonMethod, self.ServerHostType, self.IsCompanion, OsTypeIdentifier.DetectOsType(), receiveCompressionType) + rasChallenge, rasChallengeKeyVerInt, summonMethod, self.ServerHostType, self.IsCompanion, OsTypeIdentifier.DetectOsType(), receiveCompressionType, deviceId) # Send! self.OctoStream.SendMsg(buf) diff --git a/octoeverywhere/octostreammsgbuilder.py b/octoeverywhere/octostreammsgbuilder.py index 521713a..213f3e9 100644 --- a/octoeverywhere/octostreammsgbuilder.py +++ b/octoeverywhere/octostreammsgbuilder.py @@ -10,7 +10,7 @@ class OctoStreamMsgBuilder: @staticmethod - def BuildHandshakeSyn(printerId, privateKey, isPrimarySession, pluginVersion, localHttpProxyPort, localIp, rsaChallenge, rasKeyVersionInt, summonMethod, serverHostType, isCompanion, osType:OsType.OsType, receiveCompressionType:DataCompression): + def BuildHandshakeSyn(printerId, privateKey, isPrimarySession, pluginVersion, localHttpProxyPort, localIp, rsaChallenge, rasKeyVersionInt, summonMethod, serverHostType, isCompanion, osType:OsType.OsType, receiveCompressionType:DataCompression, deviceId:str): # Get a buffer builder = OctoStreamMsgBuilder.CreateBuffer(500) @@ -19,8 +19,11 @@ def BuildHandshakeSyn(printerId, privateKey, isPrimarySession, pluginVersion, lo privateKeyOffset = builder.CreateString(privateKey) pluginVersionOffset = builder.CreateString(pluginVersion) localIpOffset = None + deviceIdOffset = None if localIp is not None: localIpOffset = builder.CreateString(localIp) + if deviceId is not None: + deviceIdOffset = builder.CreateString(deviceId) # Setup the data vectors rasChallengeOffset = builder.CreateByteVector(rsaChallenge) @@ -41,6 +44,8 @@ def BuildHandshakeSyn(printerId, privateKey, isPrimarySession, pluginVersion, lo HandshakeSyn.AddRasChallengeVersion(builder, rasKeyVersionInt) HandshakeSyn.AddOsType(builder, osType) HandshakeSyn.AddReceiveCompressionType(builder, receiveCompressionType) + if deviceIdOffset is not None: + HandshakeSyn.AddDeviceId(builder, deviceIdOffset) synOffset = HandshakeSyn.End(builder) return OctoStreamMsgBuilder.CreateOctoStreamMsgAndFinalize(builder, MessageContext.MessageContext.HandshakeSyn, synOffset) diff --git a/octoeverywhere/serverauth.py b/octoeverywhere/serverauth.py index 72a3878..07c2e1d 100644 --- a/octoeverywhere/serverauth.py +++ b/octoeverywhere/serverauth.py @@ -1,4 +1,4 @@ -import random +import secrets import string import rsa @@ -30,7 +30,7 @@ def __init__(self, logger): self.Logger = logger # Generate our random challenge string. - self.Challenge = ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(ServerAuthHelper.c_ServerAuthChallengeLength)) + self.Challenge = ''.join(secrets.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(ServerAuthHelper.c_ServerAuthChallengeLength)) # Returns a string that is our challenge encrypted with the public RSA key. def GetEncryptedChallenge(self): diff --git a/octoprint_octoeverywhere/__init__.py b/octoprint_octoeverywhere/__init__.py index 5ffc765..ec3b379 100644 --- a/octoprint_octoeverywhere/__init__.py +++ b/octoprint_octoeverywhere/__init__.py @@ -16,6 +16,7 @@ from octoeverywhere.octopingpong import OctoPingPong from octoeverywhere.compression import Compression from octoeverywhere.telemetry import Telemetry +from octoeverywhere.deviceid import DeviceId from octoeverywhere.sentry import Sentry from octoeverywhere.mdns import MDns from octoeverywhere.hostcommon import HostCommon @@ -186,6 +187,9 @@ def on_startup(self, host, port): # Init the mdns helper MDns.Init(self._logger, self.get_plugin_data_folder()) + # Init device id + DeviceId.Init(self._logger) + # Init the print info manager. PrintInfoManager.Init(self._logger, self.get_plugin_data_folder()) diff --git a/octoprint_octoeverywhere/__main__.py b/octoprint_octoeverywhere/__main__.py index 1d1092f..4c6e6c9 100644 --- a/octoprint_octoeverywhere/__main__.py +++ b/octoprint_octoeverywhere/__main__.py @@ -1,7 +1,7 @@ import logging import signal import sys -import random +import secrets import string from octoeverywhere.Webcam.webcamhelper import WebcamHelper @@ -11,6 +11,7 @@ from octoeverywhere.octopingpong import OctoPingPong from octoeverywhere.compression import Compression from octoeverywhere.telemetry import Telemetry +from octoeverywhere.deviceid import DeviceId from octoeverywhere.sentry import Sentry from octoeverywhere.mdns import MDns from octoeverywhere.notificationshandler import NotificationsHandler @@ -146,7 +147,7 @@ def SignalHandler(sig, frame): sys.exit(0) def GeneratePrinterId(): - return ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(40)) + return ''.join(secrets.choice(string.ascii_uppercase + string.digits) for _ in range(40)) if __name__ == '__main__': @@ -177,6 +178,9 @@ def GeneratePrinterId(): MDns.Init(logger, PluginFilePathRoot) #MDns.Get().Test() + # Init device id + DeviceId.Init(logger) + # This is a tool to help track stuck or leaked threads. #threadDebugger = ThreadDebug() #threadDebugger.Start(logger, 30) diff --git a/octoprint_octoeverywhere/localauth.py b/octoprint_octoeverywhere/localauth.py index f73b404..aebe5d0 100644 --- a/octoprint_octoeverywhere/localauth.py +++ b/octoprint_octoeverywhere/localauth.py @@ -1,4 +1,4 @@ -import random +import secrets import string from octoprint.access.permissions import Permissions @@ -42,7 +42,7 @@ def __init__(self, logger, userManager): # But it will never call ValidateApiKey anyways. self.OctoPrintUserManager = userManager # Create a new random API key each time OctoPrint is started so we don't have to write it to disk and it changes over time. - self.ApiKey = ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(LocalAuth._ApiGeneratedKeyLength)) + self.ApiKey = ''.join(secrets.choice(string.ascii_uppercase + string.digits) for _ in range(LocalAuth._ApiGeneratedKeyLength)) # Used only for testing without actual OctoPrint, this can set the API key diff --git a/setup.py b/setup.py index 1699616..c430585 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this single version string is used by all of the plugins in OctoEverywhere! -plugin_version = "3.4.5" +plugin_version = "3.4.6" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 0e08a9fd0b6cb59c2ba9aeb46b1327ac4b13de57 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Wed, 7 Aug 2024 22:01:03 -0700 Subject: [PATCH 133/328] Making major perf changes to make things faster and help low power devices! --- .vscode/settings.json | 8 + bambu_octoeverywhere/bambuhost.py | 4 + moonraker_octoeverywhere/moonrakerclient.py | 19 +- moonraker_octoeverywhere/moonrakerhost.py | 4 + .../moonrakerwebcamhelper.py | 71 ++--- moonraker_octoeverywhere/uiinjector.py | 10 +- octoeverywhere/WebStream/octoheaderimpl.py | 6 +- octoeverywhere/WebStream/octowebstream.py | 15 +- .../WebStream/octowebstreamhttphelper.py | 12 +- octoeverywhere/debugprofiler.py | 246 ++++++++++++++++++ octoeverywhere/finalsnap.py | 16 +- octoeverywhere/gadget.py | 15 +- octoeverywhere/httpsessions.py | 84 ++++++ octoeverywhere/notificationshandler.py | 124 ++++----- octoeverywhere/octohttprequest.py | 45 +++- octoeverywhere/octopingpong.py | 129 +++++---- octoeverywhere/telemetry.py | 5 +- octoeverywhere/websocketimpl.py | 2 +- octoprint_octoeverywhere/__init__.py | 4 + octoprint_octoeverywhere/__main__.py | 4 + setup.py | 2 +- 21 files changed, 624 insertions(+), 201 deletions(-) create mode 100644 octoeverywhere/debugprofiler.py create mode 100644 octoeverywhere/httpsessions.py diff --git a/.vscode/settings.json b/.vscode/settings.json index d7f0818..d48065b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -43,6 +43,7 @@ "crypo", "Damerell", "dbus", + "debugprofiler", "decompressor", "decompressors", "deps", @@ -82,6 +83,7 @@ "hostid", "hostnames", "Hotend", + "httpsessions", "httpx", "INET", "inited", @@ -108,6 +110,7 @@ "localip", "Mailsail", "mainsailconfighandler", + "maxdepth", "mbps", "mdns", "microreads", @@ -127,6 +130,7 @@ "msgcount", "msys", "multicam", + "muppy", "myprinter", "nbsp", "networksearch", @@ -165,6 +169,7 @@ "oprint", "ostype", "ostypeidentifier", + "overhere", "paho", "peasy", "permissioned", @@ -184,7 +189,9 @@ "Pursa", "pushall", "pushd", + "pyinstrument", "Pylint", + "pympler", "pypi", "pythoncompat", "pythonhosted", @@ -194,6 +201,7 @@ "rdataclass", "rdclass", "realpath", + "refbrowser", "referer", "releaseinfo", "repeattimer", diff --git a/bambu_octoeverywhere/bambuhost.py b/bambu_octoeverywhere/bambuhost.py index eef5d1c..34eefce 100644 --- a/bambu_octoeverywhere/bambuhost.py +++ b/bambu_octoeverywhere/bambuhost.py @@ -8,6 +8,7 @@ from octoeverywhere.hostcommon import HostCommon from octoeverywhere.compression import Compression from octoeverywhere.printinfo import PrintInfoManager +from octoeverywhere.httpsessions import HttpSessions from octoeverywhere.Webcam.webcamhelper import WebcamHelper from octoeverywhere.octopingpong import OctoPingPong from octoeverywhere.commandhandler import CommandHandler @@ -71,6 +72,9 @@ def RunBlocking(self, configPath, localStorageDir, repoRoot, devConfig_CanBeNone pluginVersionStr = Version.GetPluginVersion(repoRoot) self.Logger.info("Plugin Version: %s", pluginVersionStr) + # Setup the HttpSession cache early, so it can be used whenever + HttpSessions.Init(self.Logger) + # As soon as we have the plugin version, setup Sentry # Enabling profiling and no filtering, since we are the only PY in this process. Sentry.Setup(pluginVersionStr, "bambu", devConfig_CanBeNone is not None, enableProfiling=True, filterExceptionsByPackage=False, restartOnCantCreateThreadBug=True) diff --git a/moonraker_octoeverywhere/moonrakerclient.py b/moonraker_octoeverywhere/moonrakerclient.py index e3b5baa..5fa4299 100644 --- a/moonraker_octoeverywhere/moonrakerclient.py +++ b/moonraker_octoeverywhere/moonrakerclient.py @@ -14,6 +14,7 @@ from octoeverywhere.websocketimpl import Client from octoeverywhere.notificationshandler import NotificationsHandler from octoeverywhere.exceptions import NoSentryReportException +from octoeverywhere.debugprofiler import DebugProfiler, DebugProfilerFeatures from linux_host.config import Config @@ -117,6 +118,7 @@ def __init__(self, logger:logging.Logger, config:Config, moonrakerConfigFilePath self.WebSocketConnected = False self.WebSocketKlippyReady = False self.WebSocketLock = threading.Lock() + self.WebSocketDebugProfiler:DebugProfiler = None # Must be created on the thread. self.WsThread = threading.Thread(target=self._WebSocketWorkerThread) self.WsThreadRunning = False self.WsThread.daemon = True @@ -495,6 +497,7 @@ def _GetWsMsgParam(self, msg, paramName): def _WebSocketWorkerThread(self): self.Logger.info("Moonraker client starting websocket connection thread.") + self.WebSocketDebugProfiler = DebugProfiler(self.Logger, DebugProfilerFeatures.MoonrakerWsThread) while True: try: # Every time we connect, call the function to update the host and port if required. @@ -716,14 +719,20 @@ def _onWsMsg(self, ws, msgBytes: bytes): # Raise again which will cause the websocket to close and reset. raise e + self.WebSocketDebugProfiler.ReportIfNeeded() + def _NonResponseMsgQueueWorker(self): try: - while True: - # Wait for a message to process. - msg = self.NonResponseMsgQueue.get() - # Process and then wait again. - self._OnWsNonResponseMessage(msg) + # The profiler will do nothing if it's not enabled. + with DebugProfiler(self.Logger, DebugProfilerFeatures.MoonrakerWsMsgThread) as profiler: + while True: + # Wait for a message to process. + msg = self.NonResponseMsgQueue.get() + # Process and then wait again. + self._OnWsNonResponseMessage(msg) + # Let the profiler report if needed + profiler.ReportIfNeeded() except Exception as e: Sentry.Exception("_NonReplyMsgQueueWorker got an exception while handing messages. Killing the websocket. ", e) self._RestartWebsocket() diff --git a/moonraker_octoeverywhere/moonrakerhost.py b/moonraker_octoeverywhere/moonrakerhost.py index 7e8e91e..20f74e8 100644 --- a/moonraker_octoeverywhere/moonrakerhost.py +++ b/moonraker_octoeverywhere/moonrakerhost.py @@ -8,6 +8,7 @@ from octoeverywhere.hostcommon import HostCommon from octoeverywhere.compression import Compression from octoeverywhere.octopingpong import OctoPingPong +from octoeverywhere.httpsessions import HttpSessions from octoeverywhere.Webcam.webcamhelper import WebcamHelper from octoeverywhere.printinfo import PrintInfoManager from octoeverywhere.commandhandler import CommandHandler @@ -87,6 +88,9 @@ def RunBlocking(self, klipperConfigDir, isCompanionMode, localStorageDir, servic pluginVersionStr = Version.GetPluginVersion(repoRoot) self.Logger.info("Plugin Version: %s", pluginVersionStr) + # Setup the HttpSession cache early, so it can be used whenever + HttpSessions.Init(self.Logger) + # As soon as we have the plugin version, setup Sentry # Enabling profiling and no filtering, since we are the only PY in this process. Sentry.Setup(pluginVersionStr, "klipper", devConfig_CanBeNone is not None, enableProfiling=True, filterExceptionsByPackage=False, restartOnCantCreateThreadBug=True) diff --git a/moonraker_octoeverywhere/moonrakerwebcamhelper.py b/moonraker_octoeverywhere/moonrakerwebcamhelper.py index b5e8ec4..31a7eda 100644 --- a/moonraker_octoeverywhere/moonrakerwebcamhelper.py +++ b/moonraker_octoeverywhere/moonrakerwebcamhelper.py @@ -6,6 +6,7 @@ import requests from octoeverywhere.sentry import Sentry +from octoeverywhere.debugprofiler import DebugProfiler, DebugProfilerFeatures from octoeverywhere.Webcam.webcamhelper import WebcamSettingItem, WebcamHelper from linux_host.config import Config @@ -148,39 +149,43 @@ def KickOffWebcamSettingsUpdate(self, forceUpdate = False): # This also keeps checking the auto settings option in the config, so we know if the user changes it. # Note that there's a moonraker command `notify_webcams_changed` we listen for. When it first we will force this update loop to run. def _WebcamSettingsUpdateWorker(self): - isFirstRun = True - while True: - try: - # Adjust the delay of our first run on plugin start. - delayTimeSec = MoonrakerWebcamHelper.c_DelayBetweenAutoSettingsCheckSec - if isFirstRun: - delayTimeSec = MoonrakerWebcamHelper.c_DelayForFirstRunAutoSettingsCheckSec - isFirstRun = False - - # Start the loop by clearing and waiting on the value. This means our first wake up will usually be either webcam activity - # or the moonraker client telling us the websocket is connected. Note we also do a shorter time on first run, so if klippy isn't - # in a ready state, we still try to check. - self.AutoSettingsWorkerEvent.clear() - self.AutoSettingsWorkerEvent.wait(delayTimeSec) - - # Force a config reload, so if the user changed this setting, we respect it. - self.Config.ReloadFromFile() - newAutoSettings = self.Config.GetBool(Config.WebcamSection, Config.WebcamAutoSettings, MoonrakerWebcamHelper.c_DefaultAutoSettings) - - # Log if the value changed. - if self.EnableAutoSettings != newAutoSettings: - self.Logger.info("Webcam auto settings detection value updated: "+str(newAutoSettings)) - self.EnableAutoSettings = newAutoSettings - - # Do an update if we should. - if self.EnableAutoSettings: - self._DoAutoSettingsUpdate() - else: - # Otherwise, update our in memory values with what's in the config. - self._ReadManuallySetValues() - - except Exception as e: - Sentry.Exception("Webcam helper - _WebcamSettingsUpdateWorker exception. ", e) + with DebugProfiler(self.Logger, DebugProfilerFeatures.MoonrakerWebcamHelper) as profiler: + isFirstRun = True + while True: + try: + # Adjust the delay of our first run on plugin start. + delayTimeSec = MoonrakerWebcamHelper.c_DelayBetweenAutoSettingsCheckSec + if isFirstRun: + delayTimeSec = MoonrakerWebcamHelper.c_DelayForFirstRunAutoSettingsCheckSec + isFirstRun = False + + # Start the loop by clearing and waiting on the value. This means our first wake up will usually be either webcam activity + # or the moonraker client telling us the websocket is connected. Note we also do a shorter time on first run, so if klippy isn't + # in a ready state, we still try to check. + self.AutoSettingsWorkerEvent.clear() + self.AutoSettingsWorkerEvent.wait(delayTimeSec) + + # Force a config reload, so if the user changed this setting, we respect it. + self.Config.ReloadFromFile() + newAutoSettings = self.Config.GetBool(Config.WebcamSection, Config.WebcamAutoSettings, MoonrakerWebcamHelper.c_DefaultAutoSettings) + + # Log if the value changed. + if self.EnableAutoSettings != newAutoSettings: + self.Logger.info("Webcam auto settings detection value updated: "+str(newAutoSettings)) + self.EnableAutoSettings = newAutoSettings + + # Do an update if we should. + if self.EnableAutoSettings: + self._DoAutoSettingsUpdate() + else: + # Otherwise, update our in memory values with what's in the config. + self._ReadManuallySetValues() + + # Report the profile time if needed. + profiler.ReportIfNeeded() + + except Exception as e: + Sentry.Exception("Webcam helper - _WebcamSettingsUpdateWorker exception. ", e) # Reads the values currently set in the config and sets them into our local settings. diff --git a/moonraker_octoeverywhere/uiinjector.py b/moonraker_octoeverywhere/uiinjector.py index 32508db..79b1920 100644 --- a/moonraker_octoeverywhere/uiinjector.py +++ b/moonraker_octoeverywhere/uiinjector.py @@ -7,6 +7,7 @@ from octoeverywhere.sentry import Sentry from octoeverywhere.ostypeidentifier import OsTypeIdentifier +from octoeverywhere.debugprofiler import DebugProfiler, DebugProfilerFeatures from octoeverywhere.Proto import OsType @@ -46,9 +47,12 @@ def __init__(self, logger:logging.Logger, oeRepoRoot:str): def _Worker(self): while True: try: - # Do our update logic before sleeping, so we activate right when the service loads. - # This function has it's own try except, so it won't throw out. - self._ExecuteOnce() + # The profiler will do nothing if it's not enabled. + with DebugProfiler(self.Logger, DebugProfilerFeatures.UiInjector): + + # Do our update logic before sleeping, so we activate right when the service loads. + # This function has it's own try except, so it won't throw out. + self._ExecuteOnce() # Now wait on our event handle. self.WorkerEvent.wait(UiInjector.c_UpdateCheckIntervalSec) diff --git a/octoeverywhere/WebStream/octoheaderimpl.py b/octoeverywhere/WebStream/octoheaderimpl.py index 2f39c76..0a330eb 100644 --- a/octoeverywhere/WebStream/octoheaderimpl.py +++ b/octoeverywhere/WebStream/octoheaderimpl.py @@ -3,6 +3,7 @@ from ..sentry import Sentry from ..octostreammsgbuilder import OctoStreamMsgBuilder from ..octohttprequest import OctoHttpRequest +from ..Proto.HttpInitialContext import HttpInitialContext # Indicates the base protocol, not if it's secure or not. class BaseProtocol: @@ -17,7 +18,7 @@ class HeaderHelper: # Called by slipstream and the main http class to gather and add required headers. @staticmethod - def GatherRequestHeaders(logger, httpInitialContextOptional, protocol) : + def GatherRequestHeaders(logger, httpInitialContextOptional:HttpInitialContext, protocol) : hostAddress = OctoHttpRequest.GetLocalhostAddress() @@ -33,6 +34,7 @@ def GatherRequestHeaders(logger, httpInitialContextOptional, protocol) : i += 1 # Get the values & validate + # These Key() and Value() calls are relatively what expensive, so we only call them once. name = OctoStreamMsgBuilder.BytesToString(header.Key()) value = OctoStreamMsgBuilder.BytesToString(header.Value()) if name is None or value is None: @@ -77,7 +79,7 @@ def GatherRequestHeaders(logger, httpInitialContextOptional, protocol) : value = "http://" + hostAddress # Add the header. (use the original case) - sendHeaders[OctoStreamMsgBuilder.BytesToString(header.Key())] = value + sendHeaders[name] = value # The `X-Forwarded-Host` tells the OctoPrint web server we are talking to what it's actual # hostname and port are. This allows it to set outbound urls and references to be correct to the right host. diff --git a/octoeverywhere/WebStream/octowebstream.py b/octoeverywhere/WebStream/octowebstream.py index aba020b..4f32c20 100644 --- a/octoeverywhere/WebStream/octowebstream.py +++ b/octoeverywhere/WebStream/octowebstream.py @@ -12,6 +12,7 @@ from ..Proto import WebStreamMsg from ..Proto import MessageContext from ..Proto import MessagePriority +from ..debugprofiler import DebugProfiler, DebugProfilerFeatures # # Represents a web stream, which is how we send http request and web socket messages. @@ -123,12 +124,14 @@ def SetClosedDueToFailedRequestConnection(self): # This is our main thread, where we will process all incoming messages. def run(self): - try: - self.mainThread() - except Exception as e: - Sentry.Exception("Exception in web stream ["+str(self.Id)+"] connect loop.", e) - traceback.print_exc() - self.OctoSession.OnSessionError(0) + # Enable the profiler if needed- it will do nothing if not enabled. + with DebugProfiler(self.Logger, DebugProfilerFeatures.WebStream): + try: + self.mainThread() + except Exception as e: + Sentry.Exception("Exception in web stream ["+str(self.Id)+"] connect loop.", e) + traceback.print_exc() + self.OctoSession.OnSessionError(0) def mainThread(self): diff --git a/octoeverywhere/WebStream/octowebstreamhttphelper.py b/octoeverywhere/WebStream/octowebstreamhttphelper.py index c97fe95..c7cba14 100644 --- a/octoeverywhere/WebStream/octowebstreamhttphelper.py +++ b/octoeverywhere/WebStream/octowebstreamhttphelper.py @@ -20,7 +20,6 @@ from ..Proto import MessageContext from ..Proto import HttpInitialContext from ..Proto import DataCompression -from ..Proto import MessagePriority from ..Proto import OeAuthAllowed from ..Proto.PathTypes import PathTypes @@ -1008,6 +1007,7 @@ def doBodyRead(self, octoHttpResult:OctoHttpRequest.Result, readSize:int): # Note, whatever size we pass in will be allocated as a buffer, filled, and then sliced. # So if we pass in a huge value, we will get a big buffer allocated. # So if we know the size, we should use it, so that the buffer allocated it the same amount that's returned. + # Also note, any improvements made here should be updated in ReadAllContentFromStreamResponse as well! data = response.raw.read(readSize) # If we got a data buffer return it. @@ -1143,11 +1143,13 @@ def shouldDoUnknownBodySizeRead(self, contentTypeLower_CanBeNone, contentLengthL # To speed up page load, we will defer lower pri requests while higher priority requests # are executing. def checkForDelayIfNotHighPri(self): + # This isn't used at all right now. + pass # Allow anything above Normal priority to always execute - if self.WebStreamOpenMsg.MsgPriority() < MessagePriority.MessagePriority.Normal: - return - # Otherwise, we want to block for a bit if there's a high pri stream processing. - self.WebStream.BlockIfHighPriStreamActive() + # if self.WebStreamOpenMsg.MsgPriority() < MessagePriority.MessagePriority.Normal: + # return + # # Otherwise, we want to block for a bit if there's a high pri stream processing. + # self.WebStream.BlockIfHighPriStreamActive() # Formatting helper. def _FormatFloat(self, value:float) -> str: diff --git a/octoeverywhere/debugprofiler.py b/octoeverywhere/debugprofiler.py new file mode 100644 index 0000000..5b2c332 --- /dev/null +++ b/octoeverywhere/debugprofiler.py @@ -0,0 +1,246 @@ +import time +import logging +from enum import Enum + + +# A list of possible features that can be profiled. +class DebugProfilerFeatures(Enum): + WebStream = 1 + NotificationHandlerEvent = 2 + FinalSnap = 3 + Gadget = 4 + MoonrakerWsThread = 5 + MoonrakerWsMsgThread = 6 + MoonrakerWebcamHelper = 7 + UiInjector = 8 + + +# A debug class that helps with profiling. +# +# This class must be created and only used on a single thread, that's how the profiler works. +# It can be used in two ways. +# One Off +# In one off, the class is created, started, ended, and done. +# Like: +# with DebugProfiler(self.Logger, DebugProfilerFeatures.UiInjector): +# Repeat +# For longer live threads, repeat can be used. +# You make the DebugProfiler and call start once, then you call ReportIfNeeded every so often. +# ReportIfNeeded will end the profile, print, and then start a new profile. +# +class DebugProfiler: + + # Note, this should only be True on dev builds! + # Also, the package pyinstrument needs to be manually installed. + _EnableProfiling = False + + + # A list of individual features that can be enabled for profiling. + # This also acts as a way to know what can be profiled easily. + # A value of... + # None = Disabled + # 0 = Run once + # int = Re-run every seconds. + _EnabledFeatures = { + # Enables all web streams to be profiled. + # The best way to do this is to get one URL you want to debug, enable it, and then only hit that URL. + # This includes Http and WS web streams! + # Ex: https://octoeverywhere.com/api/printer/snapshot?id= + # https://octoeverywhere.com/api/live/stream?id=. + # https://klipper.octoeverywhere.com/assets/index-17a5ec1d.js + DebugProfilerFeatures.WebStream : 0, + + # Threads used to handle notification system events. + DebugProfilerFeatures.NotificationHandlerEvent: 0, + + # The thread used for final snap. + DebugProfilerFeatures.FinalSnap: 10, + + # The thread used for final gadget. + DebugProfilerFeatures.Gadget: 10, + + # + # Klipper Only + # + + # The Moonraker main ws thread that handles the WS connection, fires message callbacks, and such. + # This usually doesn't do much, since it dispatches messages off to other threads quickly. + #DebugProfilerFeatures.MoonrakerWsThread: 10, + + # The Moonraker main thread that handles any unmatched command message. + # This does the work to handle events like print stop, start, etc. + #DebugProfilerFeatures.MoonrakerWsMsgThread: 10, + + # The webcam helper thread + # To make this fire, there must be webcam changes or something to spin the thread. + #DebugProfilerFeatures.MoonrakerWebcamHelper: 10, + + # Used to profile the UiInjector + #DebugProfilerFeatures.UiInjector : 10, + } + + + def __init__(self, logger:logging.Logger, feature:DebugProfilerFeatures, disableAutoStart = False) -> None: + self.Logger = logger + self.Feature = feature + self.Profiler = None + self.HasRan = False + self.StartedSec = 0.0 + if disableAutoStart is False: + self.StartProfile() + + + # Support using for easy integration. + def __enter__(self): + self.StartProfile() + return self + + # Support using for easy integration. + + def __exit__(self, exc_type, exc_value, traceback): + self.StopProfile() + + + # Starts the profile, only needed if you disabled auto start. + def StartProfile(self, force=False) -> None: + try: + # Ensure the profiler is enabled. + if self._EnsureEnabled() is False: + return + + # Only let this class run once. + if self.HasRan and force is False: + return + self.HasRan = True + + # Setup the profiler + # pylint: disable=import-error + # pylint: disable=import-outside-toplevel + self.Logger.info(f"Profiler started for {self.Feature}") + from pyinstrument import Profiler + self.Profiler = Profiler() + self.Profiler.start() + self.StartedSec = time.time() + except Exception as e: + self.Logger.error(f"Failed to start profiler: {e}") + + + def StopProfile(self) -> None: + try: + # Ensure the profiler is enabled. + if self._EnsureEnabled() is False: + return + + self.Logger.info(f"Profiler stopping for {self.Feature}") + self.Profiler.stop() + self.Profiler.print(unicode=True, color=True, show_all=True) + except Exception as e: + self.Logger.error(f"Failed to stop profiler: {e}") + + + def ReportIfNeeded(self) -> None: + try: + # Ensure the profiler is enabled. + if self._EnsureEnabled() is False: + return + + # Get the time to repeat. + repeatTimeSec = DebugProfiler._EnabledFeatures.get(self.Feature, None) + if repeatTimeSec is None or repeatTimeSec < 1: + self.Logger.error(f"Debug profiler ReportIfNeeded called on a feature not setup for repeat profiling. {self.Feature}") + return + + # Return if it's not time yet. + if time.time() - self.StartedSec < repeatTimeSec: + return + + # Stop and restart the profile + self.Logger.info(f"Profiler reporting for {self.Feature}") + self.StopProfile() + self.StartProfile(force=True) + except Exception as e: + self.Logger.error(f"Failed to stop profiler: {e}") + return + + + def _EnsureEnabled(self) -> bool: + if DebugProfiler._EnableProfiling is False: + return False + if DebugProfiler._EnabledFeatures.get(self.Feature, None) is None: + return False + return True + + +class MemoryProfiler(): + + # Note, this should only be True on dev builds! + # Also, the package pyinstrument needs to be manually installed. + _EnableProfiling = False + + + def __init__(self, logger:logging.Logger) -> None: + self.Logger = logger + self.Tracker = None + self._TakeMemoryProfileSnapshot() + + + def PrintMemoryDiff(self) -> None: + try: + # Ensure the profiler is enabled. + if self._EnableProfiling is False: + return + + # Print the diff. + self.Tracker.print_diff() + except Exception as e: + self.Logger.error(f"Failed to start memory profiler: {e}") + + + def PrintAllObjectsSummary(self) -> None: + try: + # Ensure the profiler is enabled. + if self._EnableProfiling is False: + return + + # pylint: disable=import-error + # pylint: disable=import-outside-toplevel + from pympler import muppy + from pympler import summary + allObjects = muppy.get_objects() + sumy = summary.summarize(allObjects) + summary.print_(sumy) + except Exception as e: + self.Logger.error(f"Failed to print all objects summary: {e}") + + + def PrintRefTreeSummary(self, rootObject) -> None: + try: + # Ensure the profiler is enabled. + if self._EnableProfiling is False: + return + + # pylint: disable=import-error + # pylint: disable=import-outside-toplevel + from pympler import refbrowser + def output_function(o): + return str(type(o)) + cb = refbrowser.ConsoleBrowser(rootObject, maxdepth=5, str_func=output_function) + cb.print_tree() + except Exception as e: + self.Logger.error(f"Failed to print ref tree summary: {e}") + + + def _TakeMemoryProfileSnapshot(self) -> None: + try: + # Ensure the profiler is enabled. + if self._EnableProfiling is False: + return + + # Setup the profiler + # pylint: disable=import-error + # pylint: disable=import-outside-toplevel + self.Logger.info("Memory profiler starting snapshot taken.") + from pympler import tracker + self.Tracker = tracker.SummaryTracker() + except Exception as e: + self.Logger.error(f"Failed to start memory profiler: {e}") diff --git a/octoeverywhere/finalsnap.py b/octoeverywhere/finalsnap.py index 02f801f..b62840e 100644 --- a/octoeverywhere/finalsnap.py +++ b/octoeverywhere/finalsnap.py @@ -5,6 +5,7 @@ from .sentry import Sentry from .repeattimer import RepeatTimer +from .debugprofiler import DebugProfiler, DebugProfilerFeatures # A helper class to try to capture a better "print completed" image by taking images before the complete notification # so we have images from shortly before the notification fires. This is needed because most printers will move the @@ -13,13 +14,15 @@ class FinalSnap: # The default interval that we will snap an image at. - c_defaultSnapIntervalSec = 1 + # We can't do this too often, or low powered devices will suffer. + c_defaultSnapIntervalSec = 2 # This is how many snapshots we will keep in our buffer. # Thus, the amount of time we will keep in our buffer is seconds = (c_snapshotBufferDepth * c_defaultSnapIntervalSec) # We must keep this buffer a little larger, for the extrude command logic to have enough buffer to operate in. # This buffer must also be large enough to have data for the c_onCompleteSnapDelaySec time. - c_snapshotBufferDepth = 40 + # This buffer can't be too large, or we will use too much memory on low end hardware + c_snapshotBufferDepth = 20 # When the on complete notification fires, this is how long we will try to go back in time to fetch a snapshot, # if we don't have a last extrude command sent time. @@ -33,6 +36,7 @@ def __init__(self, logger:logging.Logger, notificationHandler) -> None: self.NotificationHandler = notificationHandler self.SnapLock = threading.Lock() self.SnapHistory = [] + self.Profiler = None self.Timer = RepeatTimer(self.Logger, FinalSnap.c_defaultSnapIntervalSec, self._snapCallback) self.Timer.start() self.Logger.info("Starting FinalSnap") @@ -97,6 +101,11 @@ def GetFinalSnapAndStop(self): # Fires when we should take a new snapshot. def _snapCallback(self): try: + # Setup the profiler, which will no-op if not enabled. + # It must be created on this thread. + if self.Profiler is None: + self.Profiler = DebugProfiler(self.Logger, DebugProfilerFeatures.FinalSnap) + # Try to get a snapshot. snapshot = self.NotificationHandler.GetNotificationSnapshot() if snapshot is None: @@ -128,5 +137,8 @@ def _snapCallback(self): # Remove the oldest image, which is the image at the end of the list. self.SnapHistory.pop() + # Report if needed + self.Profiler.ReportIfNeeded() + except Exception as e: Sentry.Exception("FinalSnap::_snapCallback failed to get snapshot.", e) diff --git a/octoeverywhere/gadget.py b/octoeverywhere/gadget.py index 73e064b..d889c7b 100644 --- a/octoeverywhere/gadget.py +++ b/octoeverywhere/gadget.py @@ -4,11 +4,11 @@ import json import logging -import requests - from .sentry import Sentry from .snapshotresizeparams import SnapshotResizeParams from .repeattimer import RepeatTimer +from .debugprofiler import DebugProfiler, DebugProfilerFeatures +from .httpsessions import HttpSessions class Gadget: @@ -32,6 +32,7 @@ def __init__(self, logger:logging.Logger, notificationHandler, printerStateInter self.PrinterStateInterface = printerStateInterface self.Lock = threading.Lock() self.Timer = None + self.Profiler = None self.DefaultProtocolAndDomain = "https://gadget-v1-oeapi.octoeverywhere.com" self.FailedConnectionAttempts = 0 @@ -173,6 +174,11 @@ def _getTimerInterval(self): def _timerCallback(self): try: + # Setup the profiler, which will no-op if not enabled. + # It must be created on this thread. + if self.Profiler is None: + self.Profiler = DebugProfiler(self.Logger, DebugProfilerFeatures.Gadget) + # Before we do anything, update the timer interval to the default, incase there's some error # and we don't update it properly. In all cases either an error should update this or the response # from the inspect call. @@ -240,7 +246,7 @@ def _timerCallback(self): # Since we are sending the snapshot, we must send a multipart form. # Thus we must use the data and files fields, the json field will not work. # Set a timeout, but make it long, so the server has time to process. - r = requests.post(gadgetApiUrl, data=args, files=files, timeout=10*60) + r = HttpSessions.GetSession(gadgetApiUrl).post(gadgetApiUrl, data=args, files=files, timeout=10*60) # Check for success. Anything but a 200 we will consider a connection failure. if r.status_code != 200: @@ -341,6 +347,9 @@ def _timerCallback(self): # Reset the failed attempts counter self.FailedConnectionAttempts = 0 + # Report if needed + self.Profiler.ReportIfNeeded() + except Exception as e: Sentry.Exception("Exception in gadget timer", e) # On any error, clear the HostLock hostname, so we hit the root domain again. diff --git a/octoeverywhere/httpsessions.py b/octoeverywhere/httpsessions.py new file mode 100644 index 0000000..35bf225 --- /dev/null +++ b/octoeverywhere/httpsessions.py @@ -0,0 +1,84 @@ +import logging +import threading +import requests + +# A common class to cache http sessions per host. +# This makes the connections more efficient as we can reuse the connections and the session isn't created every time. +class HttpSessions: + + _Instance = None + + @staticmethod + def Init(logger:logging.Logger): + HttpSessions._Instance = HttpSessions(logger) + + + @staticmethod + def Get(): + return HttpSessions._Instance + + + def __init__(self, logger:logging.Logger): + self.Logger = logger + self.Sessions = {} + self.SessionsLock = threading.Lock() + + + # Returns a Session given the url or host. + # If the url is relative, it can be passed directly. + # If the url is absolute, the host will be extracted and used. + @staticmethod + def GetSession(hostOrUrl:str) -> requests.Session: + #pylint: disable=protected-access + return HttpSessions.Get()._GetSession(hostOrUrl) + + + def _GetSession(self, hostOrUrl:str) -> requests.Session: + # Get the root host from what's passed. + host = "" + if hostOrUrl.startswith('/'): + # There's no way to specify a port, so all relative urls are assumed to be on the same host. + host = "relative" + else: + # Extract only the host. + # Examples can be: + # https://127.0.0.1/ + # http://127.0.0.1 + # http://test.local:80/path + # ws://test.local:80/path + protocolStart = hostOrUrl.find("://") + if protocolStart == -1: + self.Logger.error("Invalid url passed to GetSession: " + hostOrUrl) + host = "unknown" + else: + # Skip past the protocol and find the host end + protocolStart += 3 + hostEnd = hostOrUrl.find("/", protocolStart) + if hostEnd == -1: + # This means the url is "http://test.local" or "http://test.local:80" + hostEnd = len(hostOrUrl) + host = hostOrUrl[:hostEnd] + + # If one exists, we don't need to lock. + s = self.Sessions.get(host, None) + if s is not None: + return s + + with self.SessionsLock: + # Check again after locking + s = self.Sessions.get(host, None) + if s is not None: + return s + + # Create a new session. + self.Logger.info(f"Creating new session for {host}") + s = requests.Session() + + # We need to be really careful of setting any params, since they will apply to all requests. + # But, from debugging we found that the requests lib takes time on every call to try to merge env vars for proxies and such. + # We don't need that, so we can just set it to False. Is saves about 20ms per request. + s.trust_env = False + + # Set the session and return it! + self.Sessions[host] = s + return s diff --git a/octoeverywhere/notificationshandler.py b/octoeverywhere/notificationshandler.py index c451980..1e5cf45 100644 --- a/octoeverywhere/notificationshandler.py +++ b/octoeverywhere/notificationshandler.py @@ -6,16 +6,16 @@ import string import logging -import requests - from .gadget import Gadget from .sentry import Sentry from .compat import Compat from .finalsnap import FinalSnap from .repeattimer import RepeatTimer +from .httpsessions import HttpSessions from .Webcam.webcamhelper import WebcamHelper from .printinfo import PrintInfoManager, PrintInfo from .snapshotresizeparams import SnapshotResizeParams +from .debugprofiler import DebugProfiler, DebugProfilerFeatures try: # On some systems this package will install but the import will fail due to a missing system .so. @@ -1037,67 +1037,69 @@ def _sendEvent(self, event:str, args = None, progressOverwriteFloat = None, useF # Sends the event # Returns True on success, otherwise False def _sendEventThreadWorker(self, event:str, args = None, progressOverwriteFloat = None, useFinalSnapSnapshot = False): - try: - # Build the common even args. - requestArgs = self.BuildCommonEventArgs(event, args, progressOverwriteFloat=progressOverwriteFloat, useFinalSnapSnapshot=useFinalSnapSnapshot) - - # Handle the result indicating we don't have the proper var to send yet. - if requestArgs is None: - self.Logger.info("NotificationsHandler didn't send the "+str(event)+" event because we don't have the proper id and key yet.") - return False - - # Break out the response - args = requestArgs[0] - files = requestArgs[1] - - # Setup the url - eventApiUrl = self.ProtocolAndDomain + "/api/printernotifications/printerevent" - - # Use fairly aggressive retry logic on notifications if they fail to send. - # This is important because they power some of the other features of OctoEverywhere now, so having them as accurate as possible is ideal. - attempts = 0 - while attempts < 6: - attempts += 1 - statusCode = 0 - try: - # Since we are sending the snapshot, we must send a multipart form. - # Thus we must use the data and files fields, the json field will not work. - r = requests.post(eventApiUrl, data=args, files=files, timeout=5*60) - - # Capture the status code. - statusCode = r.status_code - - # Check for success. - if statusCode == 200: - self.Logger.info("NotificationsHandler successfully sent '"+event+"'") - return True - - except Exception as e: - # We must try catch the connection because sometimes it will throw for some connection issues, like DNS errors, server not connectable, etc. - self.Logger.warn("Failed to send notification due to a connection error. "+str(e)) - - # On failure, log the issue. - self.Logger.warn(f"NotificationsHandler failed to send event {str(event)}. Code:{str(statusCode)}. Waiting and then trying again.") - - # If the error is in the 400 class, don't retry since these are all indications there's something - # wrong with the request, which won't change. But we don't want to include anything above or below that. - if statusCode > 399 and statusCode < 500: + # The profiler will do nothing if it's not enabled. + with DebugProfiler(self.Logger, DebugProfilerFeatures.NotificationHandlerEvent): + try: + # Build the common even args. + requestArgs = self.BuildCommonEventArgs(event, args, progressOverwriteFloat=progressOverwriteFloat, useFinalSnapSnapshot=useFinalSnapSnapshot) + + # Handle the result indicating we don't have the proper var to send yet. + if requestArgs is None: + self.Logger.info("NotificationsHandler didn't send the "+str(event)+" event because we don't have the proper id and key yet.") return False - # We have quite a few reties and back off a decent amount. As said above, we want these to be reliable as possible, even if they are late. - # We want the first few retires to be quick, so the notifications happens ASAP. This will help in teh case where the server is updating, it should be - # back withing 2-4 seconds, but 20 is a good time to wait. - # If it's still failing, we want to allow the system some time to do a do a fail over or something, thus we give the retry timer more time. - if attempts < 3: # Attempt 1 and 2 will wait 20 seconds. - time.sleep(20) - else: # Attempt 3, 4, 5 will wait longer. - time.sleep(60 * attempts) - - # We never sent it successfully. - self.Logger.error("NotificationsHandler failed to send event "+str(event)+" due to a network issues after many retries.") - - except Exception as e: - Sentry.Exception("NotificationsHandler failed to send event code "+str(event), e) + # Break out the response + args = requestArgs[0] + files = requestArgs[1] + + # Setup the url + eventApiUrl = self.ProtocolAndDomain + "/api/printernotifications/printerevent" + + # Use fairly aggressive retry logic on notifications if they fail to send. + # This is important because they power some of the other features of OctoEverywhere now, so having them as accurate as possible is ideal. + attempts = 0 + while attempts < 6: + attempts += 1 + statusCode = 0 + try: + # Since we are sending the snapshot, we must send a multipart form. + # Thus we must use the data and files fields, the json field will not work. + r = HttpSessions.GetSession(eventApiUrl).post(eventApiUrl, data=args, files=files, timeout=5*60) + + # Capture the status code. + statusCode = r.status_code + + # Check for success. + if statusCode == 200: + self.Logger.info("NotificationsHandler successfully sent '"+event+"'") + return True + + except Exception as e: + # We must try catch the connection because sometimes it will throw for some connection issues, like DNS errors, server not connectable, etc. + self.Logger.warn("Failed to send notification due to a connection error. "+str(e)) + + # On failure, log the issue. + self.Logger.warn(f"NotificationsHandler failed to send event {str(event)}. Code:{str(statusCode)}. Waiting and then trying again.") + + # If the error is in the 400 class, don't retry since these are all indications there's something + # wrong with the request, which won't change. But we don't want to include anything above or below that. + if statusCode > 399 and statusCode < 500: + return False + + # We have quite a few reties and back off a decent amount. As said above, we want these to be reliable as possible, even if they are late. + # We want the first few retires to be quick, so the notifications happens ASAP. This will help in teh case where the server is updating, it should be + # back withing 2-4 seconds, but 20 is a good time to wait. + # If it's still failing, we want to allow the system some time to do a do a fail over or something, thus we give the retry timer more time. + if attempts < 3: # Attempt 1 and 2 will wait 20 seconds. + time.sleep(20) + else: # Attempt 3, 4, 5 will wait longer. + time.sleep(60 * attempts) + + # We never sent it successfully. + self.Logger.error("NotificationsHandler failed to send event "+str(event)+" due to a network issues after many retries.") + + except Exception as e: + Sentry.Exception("NotificationsHandler failed to send event code "+str(event), e) return False diff --git a/octoeverywhere/octohttprequest.py b/octoeverywhere/octohttprequest.py index 3b10f6c..02155b3 100644 --- a/octoeverywhere/octohttprequest.py +++ b/octoeverywhere/octohttprequest.py @@ -3,10 +3,11 @@ import requests -from .localip import LocalIpHelper -from .octostreammsgbuilder import OctoStreamMsgBuilder from .mdns import MDns from .compat import Compat +from .localip import LocalIpHelper +from .httpsessions import HttpSessions +from .octostreammsgbuilder import OctoStreamMsgBuilder from .Proto.PathTypes import PathTypes from .Proto.DataCompression import DataCompression @@ -157,15 +158,39 @@ def ReadAllContentFromStreamResponse(self, logger:logging.Logger) -> None: if self._requestLibResponseObj is None: raise Exception("ReadAllContentFromStreamResponse was called on a result with no request lib Response object.") buffer = None - # We can't simply use response.content, since streaming was enabled. - # We need to use iter_content, since it will keep returning data until all is read. - # We use a high chunk count, so most of the time it will read all of the content in one go. + + # In the past, we used iter_content, but it has a lot of overhead and also doesn't read all available data, it will only read a chunk if the transfer encoding is chunked. + # This isn't great because it's slow and also we don't need to reach each chunk, process it, just to dump it in a buffer and read another. + # + # For more comments, read doBodyRead, but using read is way more efficient. + # The only other thing to note is that read will allocate the full buffer size passed, even if only some of it is filled. try: - for chunk in self._requestLibResponseObj.iter_content(10000000): + # Ideally we use the content size, but if we can't we use our default. + # The default size is tuned to fit about one 1080 jpeg image. + # Since this function is mostly used for snapshots, that's a good default. + perReadSizeBytes = 490 * 1024 + contentLengthStr = self._requestLibResponseObj.headers.get("Content-Length", None) + if contentLengthStr is not None: + perReadSizeBytes = int(contentLengthStr) + + while True: + # Read data + data = self._requestLibResponseObj.raw.read(perReadSizeBytes) + + # Check if we are done. + if data is None or len(data) == 0: + # This is weird, but there can be lingering data in response.content, so add that if there is any. + # See doBodyRead for more details. + if len(self._requestLibResponseObj.content) > 0: + buffer += self._requestLibResponseObj.content + # Break out when we are done. + break + + # If we aren't done, append the buffer. if buffer is None: - buffer = chunk + buffer = data else: - buffer += chunk + buffer += data except Exception as e: lengthStr = "[buffer is None]" if buffer is None else str(len(buffer)) logger.warn(f"ReadAllContentFromStreamResponse got an exception. We will return the current buffer length of {lengthStr}, exception: {e}") @@ -417,7 +442,7 @@ def MakeHttpCallAttempt(logger, attemptName, method, url, headers, data, mainRes # This means that response.content will not be valid and we will always use the iter_content. But it also means # iter_content will ready into memory on demand and throw when the stream is consumed. This is important, because # our logic relies on the exception when the stream is consumed to end the http response stream. - response = requests.request(method, url, headers=headers, data=data, timeout=1800, allow_redirects=allowRedirects, stream=True, verify=False) + response = HttpSessions.GetSession(url).request(method, url, headers=headers, data=data, timeout=1800, allow_redirects=allowRedirects, stream=True, verify=False) except Exception as e: logger.info(attemptName + " http URL threw an exception: "+str(e)) @@ -432,7 +457,7 @@ def MakeHttpCallAttempt(logger, attemptName, method, url, headers, data, mainRes else: logger.warn(url + " http call returned no response on Windows. Trying again with no headers.") try: - response = requests.request(method, url, headers={}, data=data, timeout=1800, allow_redirects=False, stream=True, verify=False) + response = HttpSessions.GetSession(url).request(method, url, headers={}, data=data, timeout=1800, allow_redirects=False, stream=True, verify=False) except Exception as e: logger.info(attemptName + " http NO HEADERS URL threw an exception: "+str(e)) diff --git a/octoeverywhere/octopingpong.py b/octoeverywhere/octopingpong.py index ca0c31f..8a4972c 100644 --- a/octoeverywhere/octopingpong.py +++ b/octoeverywhere/octopingpong.py @@ -298,73 +298,68 @@ def _DoPing(self, subdomain): # We have to make two calls, because the first call will query DNS, open the TCP connection, start SSL, and get a connection in the pool. # The extra stuff above will add an extra 100-150 more MS to the call. # For the first call we hit the actual API to get data back. - s = requests.Session() - response = s.get(pingInfoApiUrl, timeout=10) - - # Check for failure - if response.status_code != 200: - return None - - # Parse and check. - obj = response.json() - if "Result" not in obj: - self.Logger.warn("OctoPingPong server response had no result obj.") - return None - if "Servers" not in obj["Result"]: - self.Logger.warn("OctoPingPong server response had no servers obj.") - return None - servers = obj["Result"]["Servers"] - if "ThisServer" not in obj["Result"]: - self.Logger.warn("OctoPingPong server response had no ThisServer obj.") - return None - thisServer = obj["Result"]["ThisServer"] - if "EnablePluginAutoLowestLatency" not in obj["Result"]: - self.Logger.warn("OctoPingPong server response had no EnablePluginAutoLowestLatency obj.") - return None - enablePluginAutoLowestLatency = obj["Result"]["EnablePluginAutoLowestLatency"] - if servers is None or len(servers) == 0: - return None - if thisServer is None: - return None - - # Close this response so the connection gets put back into the pool - response.close() - - # Now using the same session, use the direct ping call. - # The session will prevent all of the overhead and should have a pooled open connection - # So this is as close to an actual realtime ping as we can get. - # - results = [] - for _ in range(0, 3): - # Do the test. - start = time.time() - response = s.get(pingDirectApiUrl, timeout=10) - end = time.time() - # Close the response so it's back in the pool. - response.close() - # Only consider 200s valid, otherwise the request might have never made it to the server. - if response.status_code == 200: - elapsedTimeMs = (end - start) * 1000.0 - results.append(elapsedTimeMs) - # Give the new test a few ms before starting again. - time.sleep(0.05) - - # Close the session to clean up all connections - # (not required, this will be auto closed, but we do it anyways) - s.close() - - # Ensure we got at least one result - if len(results) == 0: - return None - - # Since the lowest time is the fastest the server responded, that's all we care about. - minElapsedTimeMs = None - for result in results: - if minElapsedTimeMs is None or result < minElapsedTimeMs: - minElapsedTimeMs = result - - # Success. - return [minElapsedTimeMs, servers, thisServer, enablePluginAutoLowestLatency] + with requests.Session() as s: + with s.get(pingInfoApiUrl, timeout=10) as response: + # Check for failure + if response.status_code != 200: + return None + + # Parse and check. + obj = response.json() + if "Result" not in obj: + self.Logger.warn("OctoPingPong server response had no result obj.") + return None + if "Servers" not in obj["Result"]: + self.Logger.warn("OctoPingPong server response had no servers obj.") + return None + servers = obj["Result"]["Servers"] + if "ThisServer" not in obj["Result"]: + self.Logger.warn("OctoPingPong server response had no ThisServer obj.") + return None + thisServer = obj["Result"]["ThisServer"] + if "EnablePluginAutoLowestLatency" not in obj["Result"]: + self.Logger.warn("OctoPingPong server response had no EnablePluginAutoLowestLatency obj.") + return None + enablePluginAutoLowestLatency = obj["Result"]["EnablePluginAutoLowestLatency"] + if servers is None or len(servers) == 0: + return None + if thisServer is None: + return None + + # Close this response so the connection gets put back into the pool + response.close() + + # Now using the same session, use the direct ping call. + # The session will prevent all of the overhead and should have a pooled open connection + # So this is as close to an actual realtime ping as we can get. + # + results = [] + for _ in range(0, 3): + # Do the test. + start = time.time() + response = s.get(pingDirectApiUrl, timeout=10) + end = time.time() + # Close the response so it's back in the pool. + response.close() + # Only consider 200s valid, otherwise the request might have never made it to the server. + if response.status_code == 200: + elapsedTimeMs = (end - start) * 1000.0 + results.append(elapsedTimeMs) + # Give the new test a few ms before starting again. + time.sleep(0.05) + + # Ensure we got at least one result + if len(results) == 0: + return None + + # Since the lowest time is the fastest the server responded, that's all we care about. + minElapsedTimeMs = None + for result in results: + if minElapsedTimeMs is None or result < minElapsedTimeMs: + minElapsedTimeMs = result + + # Success. + return [minElapsedTimeMs, servers, thisServer, enablePluginAutoLowestLatency] except Exception as e: self.Logger.info("Failed to call _DoPing "+str(e)) diff --git a/octoeverywhere/telemetry.py b/octoeverywhere/telemetry.py index aef6957..9528ecd 100644 --- a/octoeverywhere/telemetry.py +++ b/octoeverywhere/telemetry.py @@ -1,7 +1,7 @@ import logging import threading -import requests +from .httpsessions import HttpSessions # A helper class for reporting telemetry. class Telemetry: @@ -50,7 +50,8 @@ def _WriteSync(measureStr:str, valueInt:int, fieldsOpt:dict=None, tagsOpt:dict=N event["Tags"] = tagsOpt # Send the event. - response = requests.post(Telemetry.ServerProtocolAndDomain+'/api/stats/v2/telemetryaccumulator', json=event, timeout=1*60) + url = Telemetry.ServerProtocolAndDomain+'/api/stats/v2/telemetryaccumulator' + response = HttpSessions.GetSession(url).post(url, json=event, timeout=1*60) # Check for success. if response.status_code == 200: diff --git a/octoeverywhere/websocketimpl.py b/octoeverywhere/websocketimpl.py index b9af175..f68ff0b 100644 --- a/octoeverywhere/websocketimpl.py +++ b/octoeverywhere/websocketimpl.py @@ -218,7 +218,7 @@ def _SendQueueThread(self): self.handleWsError(e) finally: # When the send queue closes, make sure the websocket is closed. - # This is a saftey, incase for some reason the websocket was open and we were told to close. + # This is a safety, incase for some reason the websocket was open and we were told to close. self._Close() diff --git a/octoprint_octoeverywhere/__init__.py b/octoprint_octoeverywhere/__init__.py index ec3b379..9d9b638 100644 --- a/octoprint_octoeverywhere/__init__.py +++ b/octoprint_octoeverywhere/__init__.py @@ -14,6 +14,7 @@ from octoeverywhere.octohttprequest import OctoHttpRequest from octoeverywhere.notificationshandler import NotificationsHandler from octoeverywhere.octopingpong import OctoPingPong +from octoeverywhere.httpsessions import HttpSessions from octoeverywhere.compression import Compression from octoeverywhere.telemetry import Telemetry from octoeverywhere.deviceid import DeviceId @@ -149,6 +150,9 @@ def on_startup(self, host, port): # Report the current setup. self._logger.info("OctoPrint host:" +str(self.OctoPrintLocalHost) + " port:" + str(self.OctoPrintLocalPort)) + # Setup the HttpSession cache early, so it can be used whenever + HttpSessions.Init(self._logger) + # Setup Sentry to capture issues. # We can't enable tracing or profiling in OctoPrint, because it picks up a lot of OctoPrint functions. Sentry.SetLogger(self._logger) diff --git a/octoprint_octoeverywhere/__main__.py b/octoprint_octoeverywhere/__main__.py index 4c6e6c9..588a1d8 100644 --- a/octoprint_octoeverywhere/__main__.py +++ b/octoprint_octoeverywhere/__main__.py @@ -9,6 +9,7 @@ from octoeverywhere.octohttprequest import OctoHttpRequest from octoeverywhere.commandhandler import CommandHandler from octoeverywhere.octopingpong import OctoPingPong +from octoeverywhere.httpsessions import HttpSessions from octoeverywhere.compression import Compression from octoeverywhere.telemetry import Telemetry from octoeverywhere.deviceid import DeviceId @@ -164,6 +165,9 @@ def GeneratePrinterId(): # Set our compat mode Compat.SetIsOctoPrint(True) + # Setup the HttpSession cache early, so it can be used whenever + HttpSessions.Init(logger) + # Init Sentry, but it won't report since we are in dev mode. Sentry.SetLogger(logger) Sentry.Setup("0.0.0", "dev", True, False) diff --git a/setup.py b/setup.py index c430585..ef9c46d 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this single version string is used by all of the plugins in OctoEverywhere! -plugin_version = "3.4.6" +plugin_version = "3.5.0" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 9de7179461d7d931803b65ebeebb413c96929513 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sun, 11 Aug 2024 12:58:05 -0700 Subject: [PATCH 134/328] Minor static UI changes --- moonraker_octoeverywhere/static/oe-ui.css | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moonraker_octoeverywhere/static/oe-ui.css b/moonraker_octoeverywhere/static/oe-ui.css index 9b6d458..ed1c4c0 100644 --- a/moonraker_octoeverywhere/static/oe-ui.css +++ b/moonraker_octoeverywhere/static/oe-ui.css @@ -54,11 +54,11 @@ } .oe-popup-button { - width: 100%; - align-self: last baseline; + flex-grow: 1; text-align: center; padding: 6px; margin-bottom: 12px; + font-size: 16px; color: white; background-color: #3F5682; From 33b56db1b6e232d71c702df27723240d241d168b Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Tue, 13 Aug 2024 19:31:37 -0700 Subject: [PATCH 135/328] Adding more docker build flavors --- .github/workflows/docker-build-test.yml | 2 +- .github/workflows/docker-publish.yml | 2 +- docker-readme.md | 6 +----- moonraker_octoeverywhere/webrequestresponsehandler.py | 2 +- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docker-build-test.yml b/.github/workflows/docker-build-test.yml index 662eedb..0380447 100644 --- a/.github/workflows/docker-build-test.yml +++ b/.github/workflows/docker-build-test.yml @@ -50,7 +50,7 @@ jobs: uses: docker/build-push-action@v5 with: context: . - platforms: linux/amd64,linux/arm/v7,linux/arm64 + platforms: linux/amd64,linux/386,linux/arm/v6,linux/arm/v7,linux/arm64 file: ./Dockerfile push: false tags: ${{ steps.meta.outputs.tags }} diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 5073606..b61dab5 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -50,7 +50,7 @@ jobs: uses: docker/build-push-action@v5 with: context: . - platforms: linux/amd64,linux/arm/v7,linux/arm64 + platforms: linux/amd64,linux/386,linux/arm/v6,linux/arm/v7,linux/arm64 file: ./Dockerfile push: true tags: ${{ steps.meta.outputs.tags }} diff --git a/docker-readme.md b/docker-readme.md index eac90b4..e44e639 100644 --- a/docker-readme.md +++ b/docker-readme.md @@ -81,14 +81,10 @@ These three values must be set at environment vars when you first run the contai - ACCESS_CODE=(code) - LAN_ONLY_MODE=TRUE -Pull the docker container locally: - -`docker pull octoeverywhere/octoeverywhere` - Run the docker container passing the required information: `docker run --name bambu-connect -e SERIAL_NUMBER= -e PRINTER_IP= -e BAMBU_CLOUD_ACCOUNT_EMAIL="" -e BAMBU_CLOUD_ACCOUNT_PASSWORD="" -v ./data:/data -d octoeverywhere/octoeverywhere` -`docker run --name bambu-connect -e SERIAL_NUMBER=test -e PRINTER_IP=1.1.1.1 -e LAN_ONLY_MODE=1 -v /data:/data -d octoeverywhere-local` +`docker run --name bambu-connect -e SERIAL_NUMBER=test -e PRINTER_IP=1.1.1.1 -e LAN_ONLY_MODE=1 -v /data:/data -d octoeverywhere/octoeverywhere` Follow the "Linking Your Bambu Connect Plugin" to link the plugin to your account. diff --git a/moonraker_octoeverywhere/webrequestresponsehandler.py b/moonraker_octoeverywhere/webrequestresponsehandler.py index 2c0ec1e..ff57d47 100644 --- a/moonraker_octoeverywhere/webrequestresponsehandler.py +++ b/moonraker_octoeverywhere/webrequestresponsehandler.py @@ -69,7 +69,7 @@ def HandleResponse(self, contextObject:ResponseHandlerContext, octoHttpResult:Oc elif contextObject.Type == ResponseHandlerContext.CameraStreamerWebRTCSdp: return self._HandleWebRtcSdpResponse(octoHttpResult, bodyBuffer) else: - self.Logger.Error("MoonrakerWebRequestResponseHandler tired to handle a context with an unknown Type? "+str(contextObject.Type)) + self.Logger.error("MoonrakerWebRequestResponseHandler tired to handle a context with an unknown Type? "+str(contextObject.Type)) except Exception as e: Sentry.Exception("MainsailConfigHandler exception while handling mainsail config.", e) return bodyBuffer From 43d7a9265ac4c5c69158de734b342e84074668c7 Mon Sep 17 00:00:00 2001 From: Yifei Liu Date: Sat, 17 Aug 2024 11:03:47 +0800 Subject: [PATCH 136/328] fix str process bug (#74) trim() is not in python, strip() instead --- docker_octoeverywhere/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker_octoeverywhere/__main__.py b/docker_octoeverywhere/__main__.py index 956a9d8..7b7b7e0 100644 --- a/docker_octoeverywhere/__main__.py +++ b/docker_octoeverywhere/__main__.py @@ -180,7 +180,7 @@ def CreateDirIfNotExists(path: str) -> None: # The region is optional. bambuCloudRegion = os.environ.get("BAMBU_CLOUD_REGION", None) if bambuCloudRegion is not None: - bambuCloudRegion = bambuCloudRegion.lower().trim() + bambuCloudRegion = bambuCloudRegion.lower().strip() if bambuCloudRegion != "china": logger.warning("The BAMBU_CLOUD_REGION should only be set to 'china' if the account is in the China region. For all other accounts it should not be set.") logger.info(f"Setting Bambu Cloud Region To: {bambuCloudRegion}") From e25701fa9399e2eff15e0a82570684742cd9ca7a Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Fri, 16 Aug 2024 20:06:58 -0700 Subject: [PATCH 137/328] Fixing various bugs and adding rekey logic. --- bambu_octoeverywhere/bambuhost.py | 27 +++++++++++++++++++++- moonraker_octoeverywhere/moonrakerhost.py | 27 +++++++++++++++++++++- octoeverywhere/Proto/HandshakeAck.py | 15 +++++++++++- octoeverywhere/Webcam/quickcam.py | 2 +- octoeverywhere/commandhandler.py | 21 +++++++++++++---- octoeverywhere/hostcommon.py | 10 ++++++++ octoeverywhere/octoservercon.py | 8 +++++++ octoeverywhere/octosessionimpl.py | 4 ++++ octoprint_octoeverywhere/__init__.py | 28 ++++++++++++++++++++++- octoprint_octoeverywhere/__main__.py | 2 +- py_installer/Installer.py | 4 ++++ 11 files changed, 137 insertions(+), 11 deletions(-) diff --git a/bambu_octoeverywhere/bambuhost.py b/bambu_octoeverywhere/bambuhost.py index 34eefce..8cb6410 100644 --- a/bambu_octoeverywhere/bambuhost.py +++ b/bambu_octoeverywhere/bambuhost.py @@ -139,7 +139,7 @@ def RunBlocking(self, configPath, localStorageDir, repoRoot, devConfig_CanBeNone stateTranslator.SetNotificationHandler(self.NotificationHandler) # Setup the command handler - CommandHandler.Init(self.Logger, self.NotificationHandler, BambuCommandHandler(self.Logger)) + CommandHandler.Init(self.Logger, self.NotificationHandler, BambuCommandHandler(self.Logger), self) # Setup the cloud if it's setup in the config. BambuCloud.Init(self.Logger, self.Config) @@ -220,6 +220,17 @@ def GetDevConfigStr(self, devConfig, value): return None + # This is a destructive action! It will remove the printer id and private key from the system and restart the plugin. + def Rekey(self, reason:str): + #pylint: disable=logging-fstring-interpolation + self.Logger.error(f"HOST REKEY CALLED {reason} - Clearing keys...") + # It's important we clear the key, or we will reload, fail to connect, try to rekey, and restart again! + self.Secrets.SetPrinterId(None) + self.Secrets.SetPrivateKey(None) + self.Logger.error("Key clear complete, restarting plugin.") + HostCommon.RestartPlugin() + + # UiPopupInvoker Interface function - Sends a UI popup message for various uses. # Must stay in sync with the OctoPrint handler! # title - string, the title text. @@ -264,3 +275,17 @@ def OnPrimaryConnectionEstablished(self, octoKey, connectedAccounts): def OnPluginUpdateRequired(self): self.Logger.error("!!! A Plugin Update Is Required -- If This Plugin Isn't Updated It Might Stop Working !!!") self.Logger.error("!!! Please use the update manager in Mainsail of Fluidd to update this plugin !!!") + + + # + # StatusChangeHandler Interface - Called by the OctoEverywhere handshake when a rekey is required. + # + def OnRekeyRequired(self): + self.Rekey("Handshake Failed") + + + # + # Command Host Interface - Called by the command handler, when called the plugin must clear it's keys and restart to generate new ones. + # + def OnRekeyCommand(self): + self.Rekey("Command") diff --git a/moonraker_octoeverywhere/moonrakerhost.py b/moonraker_octoeverywhere/moonrakerhost.py index 20f74e8..0ca9865 100644 --- a/moonraker_octoeverywhere/moonrakerhost.py +++ b/moonraker_octoeverywhere/moonrakerhost.py @@ -188,7 +188,7 @@ def RunBlocking(self, klipperConfigDir, isCompanionMode, localStorageDir, servic FileMetadataCache.Init(self.Logger, MoonrakerClient.Get()) # Setup the command handler - CommandHandler.Init(self.Logger, MoonrakerClient.Get().GetNotificationHandler(), MoonrakerCommandHandler(self.Logger)) + CommandHandler.Init(self.Logger, MoonrakerClient.Get().GetNotificationHandler(), MoonrakerCommandHandler(self.Logger), self) # If we have a local dev server, set it in the notification handler. if DevLocalServerAddress_CanBeNone is not None: @@ -281,6 +281,17 @@ def GetDevConfigStr(self, devConfig, value): return None + # This is a destructive action! It will remove the printer id and private key from the system and restart the plugin. + def Rekey(self, reason:str): + #pylint: disable=logging-fstring-interpolation + self.Logger.error(f"HOST REKEY CALLED {reason} - Clearing keys...") + # It's important we clear the key, or we will reload, fail to connect, try to rekey, and restart again! + self.Secrets.SetPrinterId(None) + self.Secrets.SetPrivateKey(None) + self.Logger.error("Key clear complete, restarting plugin.") + HostCommon.RestartPlugin() + + # # StatusChangeHandler Interface - Called by the OctoEverywhere logic when the server connection has been established. # @@ -315,6 +326,13 @@ def OnPluginUpdateRequired(self): self.Logger.error("!!! Please use the update manager in Mainsail of Fluidd to update this plugin !!!") + # + # StatusChangeHandler Interface - Called by the OctoEverywhere handshake when a rekey is required. + # + def OnRekeyRequired(self): + self.Rekey("Handshake Failed") + + # # MoonrakerClient ConnectionStatusHandler Interface - Called by the MoonrakerClient every time the moonraker websocket is open and authed - BUT possibly not connected to klippy. # At this point it's ok to query things in moonraker like db items, webcam info, and such. But API calls that have to do with the physical printer will fail, since klippy might not be ready yet. @@ -341,3 +359,10 @@ def OnWebcamSettingsChanged(self): # def OnMoonrakerClientConnected(self): pass + + + # + # Command Host Interface - Called by the command handler, when called the plugin must clear it's keys and restart to generate new ones. + # + def OnRekeyCommand(self): + self.Rekey("Command") diff --git a/octoeverywhere/Proto/HandshakeAck.py b/octoeverywhere/Proto/HandshakeAck.py index f909da7..01139f6 100644 --- a/octoeverywhere/Proto/HandshakeAck.py +++ b/octoeverywhere/Proto/HandshakeAck.py @@ -85,8 +85,15 @@ def RsaChallengeResult(self) -> Optional[str]: return self._tab.String(o + self._tab.Pos) return None + # HandshakeAck + def RequiresRekey(self): + o = octoflatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(18)) + if o != 0: + return bool(self._tab.Get(octoflatbuffers.number_types.BoolFlags, o + self._tab.Pos)) + return False + def HandshakeAckStart(builder: octoflatbuffers.Builder): - builder.StartObject(7) + builder.StartObject(8) def Start(builder: octoflatbuffers.Builder): HandshakeAckStart(builder) @@ -139,6 +146,12 @@ def HandshakeAckAddRsaChallengeResult(builder: octoflatbuffers.Builder, rsaChall def AddRsaChallengeResult(builder: octoflatbuffers.Builder, rsaChallengeResult: int): HandshakeAckAddRsaChallengeResult(builder, rsaChallengeResult) +def HandshakeAckAddRequiresRekey(builder: octoflatbuffers.Builder, requiresRekey: bool): + builder.PrependBoolSlot(7, requiresRekey, 0) + +def AddRequiresRekey(builder: octoflatbuffers.Builder, requiresRekey: bool): + HandshakeAckAddRequiresRekey(builder, requiresRekey) + def HandshakeAckEnd(builder: octoflatbuffers.Builder) -> int: return builder.EndObject() diff --git a/octoeverywhere/Webcam/quickcam.py b/octoeverywhere/Webcam/quickcam.py index 5df057a..b1b8f46 100644 --- a/octoeverywhere/Webcam/quickcam.py +++ b/octoeverywhere/Webcam/quickcam.py @@ -400,7 +400,7 @@ def GetImage(self) -> bytearray: # If the expected image size is 0, then this is the first read of 16 bytes for the header. if self.ExpectedImageSize == 0: if len(data) != 16: - raise Exception("QuickCam capture thread got a first payload that was longer than 16.") + raise Exception(f"QuickCam capture thread got a first payload that was not 16 bytes. len:{len(data)}, bytes:{data.hex()}") self.ExpectedImageSize = int.from_bytes(data[0:3], byteorder='little') # Otherwise, we are building an image else: diff --git a/octoeverywhere/commandhandler.py b/octoeverywhere/commandhandler.py index 051c671..65e5864 100644 --- a/octoeverywhere/commandhandler.py +++ b/octoeverywhere/commandhandler.py @@ -52,8 +52,8 @@ class CommandHandler: @staticmethod - def Init(logger, notificationHandler, platCommandHandler): - CommandHandler._Instance = CommandHandler(logger, notificationHandler, platCommandHandler) + def Init(logger, notificationHandler, platCommandHandler, hostCommandHandler): + CommandHandler._Instance = CommandHandler(logger, notificationHandler, platCommandHandler, hostCommandHandler) @staticmethod @@ -61,10 +61,11 @@ def Get(): return CommandHandler._Instance - def __init__(self, logger, notificationHandler, platCommandHandler): + def __init__(self, logger, notificationHandler, platCommandHandler, hostCommandHandler): self.Logger = logger self.NotificationHandler = notificationHandler self.PlatformCommandHandler = platCommandHandler + self.HostCommandHandler = hostCommandHandler # @@ -306,6 +307,15 @@ def Cancel(self): return self.PlatformCommandHandler.ExecuteCancel() + def Rekey(self): + self.Logger.warn("Rekey command received!") + resultBool = self.HostCommandHandler.OnRekeyCommand() + if resultBool: + return CommandResponse.Success() + else: + return CommandResponse.Error(400, "Failed to process rekey command.") + + # # Common Handler Core Logic # @@ -410,7 +420,8 @@ def ProcessCommand(self, commandPath, jsonObj_CanBeNone): return self.Resume() elif commandPathLower.startswith("cancel"): return self.Cancel() - + elif commandPathLower.startswith("rekey"): + return self.Rekey() return CommandResponse.Error(CommandHandler.c_CommandError_UnknownCommand, "The command path didn't match any known commands.") @@ -418,7 +429,7 @@ def ProcessCommand(self, commandPath, jsonObj_CanBeNone): class CommandResponse(): @staticmethod - def Success(resultDict:dict): + def Success(resultDict:dict = None): if resultDict is None: resultDict = {} return CommandResponse(200, resultDict, None) diff --git a/octoeverywhere/hostcommon.py b/octoeverywhere/hostcommon.py index f1edd1e..9255db8 100644 --- a/octoeverywhere/hostcommon.py +++ b/octoeverywhere/hostcommon.py @@ -1,3 +1,4 @@ +import os import string import secrets @@ -47,3 +48,12 @@ def GetAddPrinterUrl(printerId, isOctoPrint): sourceGetArg = "isFromKlipper=true" # Note this must have at least one ? and arg because users of it might append &source=blah return HostCommon.c_OctoEverywhereAddPrinterUrl + "?" + sourceGetArg + "&" + "printerid=" + printerId + + + # This will restart the plugin or if running in OctoPrint restart OctoPrint! + # Only use if absolutely needed! + @staticmethod + def RestartPlugin(): + # Use os exit, to ensure the process is killed and restarted. + # pylint: disable=protected-access + os._exit(0) diff --git a/octoeverywhere/octoservercon.py b/octoeverywhere/octoservercon.py index be72ac1..b4e1142 100644 --- a/octoeverywhere/octoservercon.py +++ b/octoeverywhere/octoservercon.py @@ -215,6 +215,14 @@ def OnPluginUpdateRequired(self): self.StatusChangeHandler.OnPluginUpdateRequired() + # Called by the server con if the plugin needs to be rekeyed. + # This is a destructive action, it will clear the printer id and private key, and force restart the plugin. + def OnRekeyRequired(self): + # This will be null for secondary connections + if self.StatusChangeHandler is not None: + self.StatusChangeHandler.OnRekeyRequired() + + # A summon request can be sent by the services if the user is connected to a different # server than we are connected to. In such a case we will multi connect a temp non-primary connection # to the request server as well, that will be to service the user. diff --git a/octoeverywhere/octosessionimpl.py b/octoeverywhere/octosessionimpl.py index 5a1cfdc..e11a117 100644 --- a/octoeverywhere/octosessionimpl.py +++ b/octoeverywhere/octosessionimpl.py @@ -150,6 +150,10 @@ def HandleHandshakeAck(self, msg): backoffModifierSec = 43200 # 1 month self.OctoStream.OnPluginUpdateRequired() + # Check if a rekey was requested, if so, the plugin needs to rekey and restart. + if handshakeAck.RequiresRekey(): + self.OctoStream.OnRekeyRequired() + self.OnSessionError(backoffModifierSec) diff --git a/octoprint_octoeverywhere/__init__.py b/octoprint_octoeverywhere/__init__.py index 9d9b638..1b08af3 100644 --- a/octoprint_octoeverywhere/__init__.py +++ b/octoprint_octoeverywhere/__init__.py @@ -206,7 +206,7 @@ def on_startup(self, host, port): printerStateObject.SetNotificationHandler(self.NotificationHandler) # Create our command handler and our platform specific command handler. - CommandHandler.Init(self._logger, self.NotificationHandler, OctoPrintCommandHandler(self._logger, self._printer, printerStateObject, self)) + CommandHandler.Init(self._logger, self.NotificationHandler, OctoPrintCommandHandler(self._logger, self._printer, printerStateObject, self), self) # Create the smart pause handler SmartPause.Init(self._logger, self._printer, self._printer_profile_manager.get_current_or_default()) @@ -694,6 +694,32 @@ def OnPluginUpdateRequired(self): self._logger.error("The OctoEverywhere service told us we must update before we can connect.") self.SetPluginUpdateRequired(True) + + # + # StatusChangeHandler Interface - Called by the OctoEverywhere handshake when a rekey is required. + # + def OnRekeyRequired(self): + self.Rekey("Handshake Failure") + + + # + # Command Host Interface - Called by the command handler, when called the plugin must clear it's keys and restart to generate new ones. + # + def OnRekeyCommand(self): + self.Rekey("Commanded") + + + # This is a destructive action! It will remove the printer id and private key from the system and restart the plugin. + def Rekey(self, reason:str): + #pylint: disable=logging-fstring-interpolation + self._logger.error(f"HOST REKEY CALLED {reason} - Clearing keys...") + # It's important we clear the key, or we will reload, fail to connect, try to rekey, and restart again! + self.SaveToSettingsIfUpdated("PrinterKey", "") + self.SaveToSettingsIfUpdated("Pid", "") + self._logger.error("Key clear complete, restarting plugin.") + HostCommon.RestartPlugin() + + # Our main worker def main(self): self._logger.info("Main thread starting") diff --git a/octoprint_octoeverywhere/__main__.py b/octoprint_octoeverywhere/__main__.py index 588a1d8..f66beee 100644 --- a/octoprint_octoeverywhere/__main__.py +++ b/octoprint_octoeverywhere/__main__.py @@ -221,7 +221,7 @@ def GeneratePrinterId(): NotificationHandlerInstance = NotificationsHandler(logger, MockPrinterStateObject(logger)) # Setup the api command handler if needed for testing. - CommandHandler.Init(logger, NotificationHandlerInstance, None) + CommandHandler.Init(logger, NotificationHandlerInstance, None, None) # Note this will throw an exception because we don't have a flask context setup. # result = apiCommandHandler.HandleApiCommand("status", None) # Setup the command handler diff --git a/py_installer/Installer.py b/py_installer/Installer.py index 1ee5272..ababf72 100644 --- a/py_installer/Installer.py +++ b/py_installer/Installer.py @@ -2,6 +2,7 @@ import traceback from octoeverywhere.telemetry import Telemetry +from octoeverywhere.httpsessions import HttpSessions from .Linker import Linker from .Logging import Logger @@ -56,6 +57,9 @@ def _RunInternal(self): # As soon as we have the user home make the log file. Logger.InitFile(context.UserHomePath, context.UserName) + + # Next we init the http sessions and telemetry, telemetry relies on the http sessions logic. + HttpSessions.Init(Logger.GetPyLogger()) Telemetry.Init(Logger.GetPyLogger()) # Parse the original CmdLineArgs From 0c33c80fde3f8a3117ef2df6cca4696a86a07ec9 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Fri, 16 Aug 2024 21:35:08 -0700 Subject: [PATCH 138/328] Adding a bed cooldown notification! --- .vscode/settings.json | 3 + bambu_octoeverywhere/bambumodels.py | 8 +- bambu_octoeverywhere/bambustatetranslater.py | 11 +++ moonraker_octoeverywhere/moonrakerclient.py | 35 +++++++ .../moonrakercommandhandler.py | 1 + octoeverywhere/notifications/__init__.py | 0 .../notifications/bedcooldownwatcher.py | 95 +++++++++++++++++++ octoeverywhere/notificationshandler.py | 15 +++ octoprint_octoeverywhere/__main__.py | 6 ++ .../printerstateobject.py | 24 +++++ 10 files changed, 194 insertions(+), 4 deletions(-) create mode 100644 octoeverywhere/notifications/__init__.py create mode 100644 octoeverywhere/notifications/bedcooldownwatcher.py diff --git a/.vscode/settings.json b/.vscode/settings.json index d48065b..fdb6093 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -20,6 +20,8 @@ "bambustatetranslater", "bambuwebcamhelper", "bblp", + "bedcooldowncomplete", + "bedcooldownwatcher", "bootstraper", "boundarydonotcross", "brotli", @@ -35,6 +37,7 @@ "companionconfigfile", "coms", "continuousprint", + "Cooldown", "Creality", "Creality's", "creds", diff --git a/bambu_octoeverywhere/bambumodels.py b/bambu_octoeverywhere/bambumodels.py index e4fda12..d3f83a5 100644 --- a/bambu_octoeverywhere/bambumodels.py +++ b/bambu_octoeverywhere/bambumodels.py @@ -25,10 +25,10 @@ def __init__(self) -> None: self.total_layer_num:int = None self.subtask_name:str = None self.mc_percent:int = None - self.nozzle_temper:int = None - self.nozzle_target_temper:int = None - self.bed_temper:int = None - self.bed_target_temper:int = None + self.nozzle_temper:float = None + self.nozzle_target_temper:float = None + self.bed_temper:float = None + self.bed_target_temper:float = None self.mc_remaining_time:int = None self.project_id:str = None self.print_error:int = None diff --git a/bambu_octoeverywhere/bambustatetranslater.py b/bambu_octoeverywhere/bambustatetranslater.py index 029ffe9..01d92b7 100644 --- a/bambu_octoeverywhere/bambustatetranslater.py +++ b/bambu_octoeverywhere/bambustatetranslater.py @@ -260,3 +260,14 @@ def IsPrintWarmingUp(self): if state.stg_cur == 1 or state.stg_cur == 2 or state.stg_cur == 7 or state.stg_cur == 9 or state.stg_cur == 11 or state.stg_cur == 14: return True return False + + + # ! Interface Function ! The entire interface must change if the function is changed. + # Returns the current hotend temp and bed temp as a float in celsius if they are available, otherwise None. + def GetTemps(self): + state = BambuClient.Get().GetState() + if state is None: + return (None, None) + + # These will be None if they are unknown. + return (state.nozzle_temper, state.bed_temper) diff --git a/moonraker_octoeverywhere/moonrakerclient.py b/moonraker_octoeverywhere/moonrakerclient.py index 5fa4299..65ef1e2 100644 --- a/moonraker_octoeverywhere/moonrakerclient.py +++ b/moonraker_octoeverywhere/moonrakerclient.py @@ -1132,6 +1132,41 @@ def IsPrintWarmingUp(self): return self.CheckIfPrinterIsWarmingUp_WithPrintStats(result) + # ! Interface Function ! The entire interface must change if the function is changed. + # Returns the current hotend temp and bed temp as a float in celsius if they are available, otherwise None. + def GetTemps(self): + result = MoonrakerClient.Get().SendJsonRpcRequest("printer.objects.query", + { + "objects": { + "extruder": None, # Needed for temps + "heater_bed": None, # Needed for temps + } + }) + # Validate + if result.HasError(): + self.Logger.error("MoonrakerCommandHandler failed GetTemps() query. "+result.GetLoggingErrorStr()) + return (None, None) + + # Get the result. + res = result.GetResult() + + # Get the current temps if possible. + # Shared code with MoonrakerCommandHandler.GetCurrentJobStatus + hotendActual = None + bedActual = None + if "status" in res and "extruder" in res["status"]: + extruder = res["status"]["extruder"] + if "temperature" in extruder: + hotendActual = round(float(extruder["temperature"]), 2) + if "status" in res and "heater_bed" in res["status"]: + heater_bed = res["status"]["heater_bed"] + if "temperature" in heater_bed: + bedActual = round(float(heater_bed["temperature"]), 2) + + return (hotendActual, bedActual) + + + # # Helpers # diff --git a/moonraker_octoeverywhere/moonrakercommandhandler.py b/moonraker_octoeverywhere/moonrakercommandhandler.py index a0af6fc..f6e153e 100644 --- a/moonraker_octoeverywhere/moonrakercommandhandler.py +++ b/moonraker_octoeverywhere/moonrakercommandhandler.py @@ -116,6 +116,7 @@ def GetCurrentJobStatus(self): timeLeftSec = MoonrakerClient.Get().GetMoonrakerCompat().GetPrintTimeRemainingEstimateInSeconds_WithPrintStatsVirtualSdCardAndGcodeMoveResult(result) # Get the current temps if possible. + # Shared code with MoonrakerClient.GetTemps hotendActual = 0.0 hotendTarget = 0.0 bedTarget = 0.0 diff --git a/octoeverywhere/notifications/__init__.py b/octoeverywhere/notifications/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/octoeverywhere/notifications/bedcooldownwatcher.py b/octoeverywhere/notifications/bedcooldownwatcher.py new file mode 100644 index 0000000..70fbbaf --- /dev/null +++ b/octoeverywhere/notifications/bedcooldownwatcher.py @@ -0,0 +1,95 @@ +import time +import logging +import threading + +from ..sentry import Sentry +from ..repeattimer import RepeatTimer + +# A simple class to watch for the bed to cooldown and then fires a notification. +class BedCooldownWatcher: + + # The amount of time between checks. + c_checkIntervalSec = 2 + + # The max amount of time we will allow this to keep watching. + c_maxWatcherRuntimeSec = 60 * 20 + + + def __init__(self, logger:logging.Logger, notificationHandler, printerStateInterface): + self.Logger = logger + self.NotificationHandler = notificationHandler + self.PrinterStateInterface = printerStateInterface + self.Timer = None + self.TimerStartSec = None + self.IsFirstTimerRead = True + self.Lock = threading.Lock() + + + # Starts the waiter if it's not running. + def Start(self) -> None: + with self.Lock: + # Stop any running timer. + self._stopTimerUnderLock() + + self.Logger.info("Bed cooldown watcher starting") + self.TimerStartSec = time.time() + self.IsFirstTimerRead = True + + # Start a new timer. + self.Timer = RepeatTimer(self.Logger, BedCooldownWatcher.c_checkIntervalSec, self._timerCallback) + self.Timer.start() + + + # Stops the timer if it's running. + def Stop(self): + with self.Lock: + self._stopTimerUnderLock() + + + + def _stopTimerUnderLock(self): + if self.Timer is not None: + self.Logger.info("Bed cooldown watcher stopped.") + self.Timer.Stop() + self.Timer = None + + + def _timerCallback(self): + try: + # Check if we should stop watching. + if time.time() - self.TimerStartSec > BedCooldownWatcher.c_maxWatcherRuntimeSec: + self.Logger.info("Bed cooldown watcher, max runtime reached. Stopping.") + self.Stop() + return + + # Try to get the current temps + (_, bedTempCelsiusFloat) = self.PrinterStateInterface.GetTemps() + if bedTempCelsiusFloat is None: + self.Logger.info("Bed cooldown watcher, no bed temp available. Stopping.") + self.Stop() + return + + isFirstTimerRead = self.IsFirstTimerRead + self.IsFirstTimerRead = False + + # When the bed is under ~90F, we will consider it cooled down. + if bedTempCelsiusFloat > 33: + # Keep waiting. + self.Logger.debug(f"Bed cooldown watcher, bed temp is {bedTempCelsiusFloat}. Waiting...") + return + + # If this is the first read and the bed is already cool, we won't notify. + if isFirstTimerRead: + self.Logger.info("Bed cooldown watcher, bed is cooled down, but it was already cool on the first read, so we won't notify.") + self.Stop() + return + + # The bed is cooled down. + self.Logger.info("Bed cooldown watcher, bed is cooled down.") + self.Stop() + + # Fire the notification. + self.NotificationHandler.OnBedCooldownComplete(bedTempCelsiusFloat) + + except Exception as e: + Sentry.Exception("BedCooldownWatcher exception in timer callback", e) diff --git a/octoeverywhere/notificationshandler.py b/octoeverywhere/notificationshandler.py index 1e5cf45..d863c40 100644 --- a/octoeverywhere/notificationshandler.py +++ b/octoeverywhere/notificationshandler.py @@ -16,6 +16,7 @@ from .printinfo import PrintInfoManager, PrintInfo from .snapshotresizeparams import SnapshotResizeParams from .debugprofiler import DebugProfiler, DebugProfilerFeatures +from .notifications.bedcooldownwatcher import BedCooldownWatcher try: # On some systems this package will install but the import will fail due to a missing system .so. @@ -60,6 +61,7 @@ def __init__(self, logger:logging.Logger, printerStateInterface): self.FirstLayerTimer = None self.FinalSnapObj:FinalSnap = None self.Gadget = Gadget(logger, self, self.PrinterStateInterface) + self.BedCooldownWatcher = BedCooldownWatcher(logger, self, self.PrinterStateInterface) # Define all the vars we use locally in the notification handler self.PrintCookie = "" @@ -349,6 +351,7 @@ def OnFailed(self, fileName:str, durationSecStr:str = None, reason:str = None): self._updateCurrentFileName(fileName) self._updateToKnownDuration(durationSecStr) self.StopTimers() + self.BedCooldownWatcher.Start() self._sendEvent("failed", { "Reason": reason}) @@ -360,6 +363,7 @@ def OnDone(self, fileName:str = None, durationSecStr:str = None): self._updateCurrentFileName(fileName) self._updateToKnownDuration(durationSecStr) self.StopTimers() + self.BedCooldownWatcher.Start() self._sendEvent("done", useFinalSnapSnapshot=True) @@ -409,6 +413,10 @@ def OnError(self, error): self.StopTimers() + # Start the cooldown watcher because on it's first check, if the bed is already cool, + # it won't fire any notifications. + self.BedCooldownWatcher.Start() + # This might be spammy from OctoPrint, so limit how often we bug the user with them. if self._shouldSendSpammyEvent("on-error"+str(error), 30.0) is False: return @@ -540,6 +548,13 @@ def OnPrintTimerProgress(self): self._sendEvent("timerprogress", { "HoursCount": str(self.PingTimerHoursReported) }) + # Called by the bed cooldown watcher when the bed is done cooling down. + def OnBedCooldownComplete(self, bedTempCelsius:float): + if self._shouldIgnoreEvent(): + return + self._sendEvent("bedcooldowncomplete", { "BedTempC": str(round(float(bedTempCelsius), 2)) }) + + # # Note this values are important! # The cost of getting the current z offset is decently high, and thus we can't check it too often. diff --git a/octoprint_octoeverywhere/__main__.py b/octoprint_octoeverywhere/__main__.py index f66beee..03932d2 100644 --- a/octoprint_octoeverywhere/__main__.py +++ b/octoprint_octoeverywhere/__main__.py @@ -108,6 +108,12 @@ def IsPrintWarmingUp(self): return False + # ! Interface Function ! The entire interface must change if the function is changed. + # Returns the current hotend temp and bed temp as a float in celsius if they are available, otherwise None. + def GetTemps(self): + return (None, None) + + # A mock of the popup UI interface. NotificationHandlerInstance = None class StatusChangeHandlerStub(): diff --git a/octoprint_octoeverywhere/printerstateobject.py b/octoprint_octoeverywhere/printerstateobject.py index c1de43b..fa3c328 100644 --- a/octoprint_octoeverywhere/printerstateobject.py +++ b/octoprint_octoeverywhere/printerstateobject.py @@ -125,3 +125,27 @@ def IsPrintWarmingUp(self): # We aren't warming up. return False + + + # ! Interface Function ! The entire interface must change if the function is changed. + # Returns the current hotend temp and bed temp as a float in celsius if they are available, otherwise None. + def GetTemps(self): + # Get the current temps if possible. + # Note there will be no objects in the dic if the printer isn't connected or in other cases. + currentTemps = self.OctoPrintPrinterObject.get_current_temperatures() + hotendActual = None + bedActual = None + if self._Exists(currentTemps, "tool0"): + tool0 = currentTemps["tool0"] + if self._Exists(tool0, "actual"): + hotendActual = round(float(tool0["actual"]), 2) + if self._Exists(currentTemps, "bed"): + bed = currentTemps["bed"] + if self._Exists(bed, "actual"): + bedActual = round(float(bed["actual"]), 2) + return (hotendActual, bedActual) + + + # A helper for checking if things exist in dicts. + def _Exists(self, dictObj:dict, key:str): + return key in dictObj and dictObj[key] is not None From 03a181dfdc8add94cbec93ad34c0d56cdb2bf2bc Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Fri, 16 Aug 2024 21:37:33 -0700 Subject: [PATCH 139/328] Version bump! --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ef9c46d..e1d6418 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this single version string is used by all of the plugins in OctoEverywhere! -plugin_version = "3.5.0" +plugin_version = "3.5.1" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 1898c764bcab7835c623f3d81e5a3893c5a46c6d Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Fri, 16 Aug 2024 22:30:29 -0700 Subject: [PATCH 140/328] Removing an annoying log. --- moonraker_octoeverywhere/moonrakerclient.py | 3 ++- setup.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/moonraker_octoeverywhere/moonrakerclient.py b/moonraker_octoeverywhere/moonrakerclient.py index 65ef1e2..4a42e14 100644 --- a/moonraker_octoeverywhere/moonrakerclient.py +++ b/moonraker_octoeverywhere/moonrakerclient.py @@ -1069,7 +1069,8 @@ def GetCurrentLayerInfo(self): if totalLayers == 0: if self.Logger.isEnabledFor(logging.DEBUG): self.Logger.debug("GetCurrentLayerInfo failed to get a total layer count. "+json.dumps(printStats)) - self.Logger.warn("GetCurrentLayerInfo failed to get a total layer count.") + # Dont log, this seems to be somewhat common on some printers like the K1 and others. + #self.Logger.warn("GetCurrentLayerInfo failed to get a total layer count.") return (0,0) # Next, try to get the current layer. diff --git a/setup.py b/setup.py index e1d6418..d065612 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this single version string is used by all of the plugins in OctoEverywhere! -plugin_version = "3.5.1" +plugin_version = "3.5.2" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From e62f2e3858cc27df2b354f34268a47d64f29ac1d Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Fri, 16 Aug 2024 22:34:36 -0700 Subject: [PATCH 141/328] Fixing one more little bed cooldown issue. --- octoeverywhere/notificationshandler.py | 3 +++ setup.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/octoeverywhere/notificationshandler.py b/octoeverywhere/notificationshandler.py index d863c40..43730e1 100644 --- a/octoeverywhere/notificationshandler.py +++ b/octoeverywhere/notificationshandler.py @@ -126,6 +126,9 @@ def _RecoverOrRestForNewPrint(self, printCookie:str): # Ensure there's no final snap running. self._getFinalSnapSnapshotAndStop() + # Ensure the bed cooldown watcher is stopped. + self.BedCooldownWatcher.Stop() + # The print cookie can only be None on class init. # We pass None so we don't call the PrintInfoManager, which might create a new print info on disk. # There might be a print info on disk we want to restore when the host connects to the printer. diff --git a/setup.py b/setup.py index d065612..c730595 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this single version string is used by all of the plugins in OctoEverywhere! -plugin_version = "3.5.2" +plugin_version = "3.5.3" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 4a83118861d60e4abd435827668d483172a015cc Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sat, 17 Aug 2024 13:13:26 -0700 Subject: [PATCH 142/328] Adding a config option to control the cooldown temp threshold. --- bambu_octoeverywhere/bambuhost.py | 1 + linux_host/config.py | 28 +++++++++++++++++++ moonraker_octoeverywhere/moonrakerclient.py | 6 ++-- .../notifications/bedcooldownwatcher.py | 14 ++++++++-- octoeverywhere/notificationshandler.py | 5 ++++ 5 files changed, 50 insertions(+), 4 deletions(-) diff --git a/bambu_octoeverywhere/bambuhost.py b/bambu_octoeverywhere/bambuhost.py index 8cb6410..4c88058 100644 --- a/bambu_octoeverywhere/bambuhost.py +++ b/bambu_octoeverywhere/bambuhost.py @@ -136,6 +136,7 @@ def RunBlocking(self, configPath, localStorageDir, repoRoot, devConfig_CanBeNone stateTranslator = BambuStateTranslator(self.Logger) self.NotificationHandler = NotificationsHandler(self.Logger, stateTranslator) self.NotificationHandler.SetPrinterId(printerId) + self.NotificationHandler.SetBedCooldownThresholdTemp(self.Config.GetFloat(Config.GeneralSection, Config.GeneralBedCooldownThresholdTempC, Config.GeneralBedCooldownThresholdTempCDefault)) stateTranslator.SetNotificationHandler(self.NotificationHandler) # Setup the command handler diff --git a/linux_host/config.py b/linux_host/config.py index 2c2c0de..6f35a7a 100644 --- a/linux_host/config.py +++ b/linux_host/config.py @@ -20,6 +20,10 @@ class Config: LogFileMaxSizeMbKey = "max_file_size_mb" LogFileMaxCountKey = "max_file_count" + GeneralSection = "general" + GeneralBedCooldownThresholdTempC = "bed_cooldown_threshold_temp_celsius" + GeneralBedCooldownThresholdTempCDefault = 40.0 + # # Used for the local Moonraker plugin and companions. @@ -79,6 +83,7 @@ class Config: { "Target": WebcamFlipH, "Comment": "Flips the webcam image horizontally. Valid values are True or False"}, { "Target": WebcamFlipV, "Comment": "Flips the webcam image vertically. Valid values are True or False"}, { "Target": WebcamRotation, "Comment": "Rotates the webcam image. Valid values are 0, 90, 180, or 270"}, + { "Target": GeneralBedCooldownThresholdTempC, "Comment": "The temperature in Celsius that the bed must be under to be considered cooled down. This is used to fire the Bed Cooldown Complete notification.."}, ] @@ -163,6 +168,29 @@ def GetInt(self, section:str, key:str, defaultValue) -> int: return int(defaultValue) + # Gets a value from the config given the header and key. + # If the value isn't set, the default value is returned and the default value is saved into the config. + # If the default value is None, the default will not be written into the config. + def GetFloat(self, section:str, key:str, defaultValue) -> float: + # Use a try catch, so if a user sets an invalid value, it doesn't crash us. + result = None + try: + # If None is passed as the default, don't str it. + if defaultValue is not None: + defaultValue = str(defaultValue) + + result = self.GetStr(section, key, defaultValue) + # If None is returned, don't int it, return None. + if result is None: + return None + + return float(result) + except Exception as e: + self.Logger.error(f"Config settings error! {key} failed to get as float. Value was `{result}`. Resetting to default. "+str(e)) + self.SetStr(section, key, str(defaultValue)) + return float(defaultValue) + + # Gets a value from the config given the header and key. # If the value isn't set, the default value is returned and the default value is saved into the config. # If the default value is None, the default will not be written into the config. diff --git a/moonraker_octoeverywhere/moonrakerclient.py b/moonraker_octoeverywhere/moonrakerclient.py index 4a42e14..34776fa 100644 --- a/moonraker_octoeverywhere/moonrakerclient.py +++ b/moonraker_octoeverywhere/moonrakerclient.py @@ -100,7 +100,8 @@ def __init__(self, logger:logging.Logger, config:Config, moonrakerConfigFilePath self.JsonRpcWaitingContexts = {} # Setup the Moonraker compat helper object. - self.MoonrakerCompat = MoonrakerCompat(self.Logger, printerId) + cooldownThresholdTempC = self.Config.GetFloat(Config.GeneralSection, Config.GeneralBedCooldownThresholdTempC, Config.GeneralBedCooldownThresholdTempCDefault) + self.MoonrakerCompat = MoonrakerCompat(self.Logger, printerId, cooldownThresholdTempC) # Setup the non response message thread # See _NonResponseMsgQueueWorker to why this is needed. @@ -786,7 +787,7 @@ def SetSocketClosed(self): # common OctoEverywhere logic. class MoonrakerCompat: - def __init__(self, logger:logging.Logger, printerId:str) -> None: + def __init__(self, logger:logging.Logger, printerId:str, bedCooldownThresholdTempC:float) -> None: self.Logger = logger # This indicates if we are ready to process notifications, so we don't @@ -800,6 +801,7 @@ def __init__(self, logger:logging.Logger, printerId:str) -> None: # We pass our self as the Printer State Interface self.NotificationHandler = NotificationsHandler(self.Logger, self) self.NotificationHandler.SetPrinterId(printerId) + self.NotificationHandler.SetBedCooldownThresholdTemp(bedCooldownThresholdTempC) def SetOctoKey(self, octoKey:str): diff --git a/octoeverywhere/notifications/bedcooldownwatcher.py b/octoeverywhere/notifications/bedcooldownwatcher.py index 70fbbaf..5cad76d 100644 --- a/octoeverywhere/notifications/bedcooldownwatcher.py +++ b/octoeverywhere/notifications/bedcooldownwatcher.py @@ -16,6 +16,11 @@ class BedCooldownWatcher: def __init__(self, logger:logging.Logger, notificationHandler, printerStateInterface): + + # Default the the bed is under ~100F, we will consider it cooled down. + # This can be changed in the config by the user. + self.CooldownThresholdTempC:float = 40.0 + self.Logger = logger self.NotificationHandler = notificationHandler self.PrinterStateInterface = printerStateInterface @@ -46,6 +51,11 @@ def Stop(self): self._stopTimerUnderLock() + # Sets the temp that the bed must be under to be considered cooled down. + def SetBedCooldownThresholdTemp(self, tempC:float): + self.Logger.debug(f"Bed cooldown watcher, setting threshold temp to {tempC}") + self.CooldownThresholdTempC = tempC + def _stopTimerUnderLock(self): if self.Timer is not None: @@ -72,8 +82,8 @@ def _timerCallback(self): isFirstTimerRead = self.IsFirstTimerRead self.IsFirstTimerRead = False - # When the bed is under ~90F, we will consider it cooled down. - if bedTempCelsiusFloat > 33: + # Check if we are cooled down yet. + if bedTempCelsiusFloat > self.CooldownThresholdTempC: # Keep waiting. self.Logger.debug(f"Bed cooldown watcher, bed temp is {bedTempCelsiusFloat}. Waiting...") return diff --git a/octoeverywhere/notificationshandler.py b/octoeverywhere/notificationshandler.py index 43730e1..6d6da19 100644 --- a/octoeverywhere/notificationshandler.py +++ b/octoeverywhere/notificationshandler.py @@ -209,6 +209,11 @@ def IsTrackingPrint(self) -> bool: return self._IsPingTimerRunning() + # Sets the cooldown threshold temp + def SetBedCooldownThresholdTemp(self, tempC:float): + self.BedCooldownWatcher.SetBedCooldownThresholdTemp(tempC) + + # A special case used by moonraker and bambu to restore the state of an ongoing print that we don't know of. # What we want to do is check moonraker or bambu's current state and our current state, to see if there's anything that needs to be synced. # Remember that we might be syncing because our service restarted during a print, or moonraker restarted, so we might already have From 2186b5a6dd7beeadb5e27c7643f9318b8d98d361 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sat, 17 Aug 2024 13:16:13 -0700 Subject: [PATCH 143/328] Minor tweaks and a version bump. --- linux_host/config.py | 2 +- octoeverywhere/notifications/bedcooldownwatcher.py | 8 +++++--- setup.py | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/linux_host/config.py b/linux_host/config.py index 6f35a7a..2a9df2a 100644 --- a/linux_host/config.py +++ b/linux_host/config.py @@ -83,7 +83,7 @@ class Config: { "Target": WebcamFlipH, "Comment": "Flips the webcam image horizontally. Valid values are True or False"}, { "Target": WebcamFlipV, "Comment": "Flips the webcam image vertically. Valid values are True or False"}, { "Target": WebcamRotation, "Comment": "Rotates the webcam image. Valid values are 0, 90, 180, or 270"}, - { "Target": GeneralBedCooldownThresholdTempC, "Comment": "The temperature in Celsius that the bed must be under to be considered cooled down. This is used to fire the Bed Cooldown Complete notification.."}, + { "Target": GeneralBedCooldownThresholdTempC, "Comment": "The temperature in Celsius that the bed must be under to be considered cooled down. This is used to fire the Bed Cooldown Complete notification."}, ] diff --git a/octoeverywhere/notifications/bedcooldownwatcher.py b/octoeverywhere/notifications/bedcooldownwatcher.py index 5cad76d..f2b6102 100644 --- a/octoeverywhere/notifications/bedcooldownwatcher.py +++ b/octoeverywhere/notifications/bedcooldownwatcher.py @@ -8,11 +8,13 @@ # A simple class to watch for the bed to cooldown and then fires a notification. class BedCooldownWatcher: - # The amount of time between checks. - c_checkIntervalSec = 2 + # The amount of time between checks in seconds. + c_checkIntervalSec = 5 # The max amount of time we will allow this to keep watching. - c_maxWatcherRuntimeSec = 60 * 20 + # In some cases like enclosed printers, the bed cooldown might take a very long time. + # We cancel this watcher when a new print starts, so it's safe to have a long runtime. + c_maxWatcherRuntimeSec = 60 * 60 def __init__(self, logger:logging.Logger, notificationHandler, printerStateInterface): diff --git a/setup.py b/setup.py index c730595..a37093f 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this single version string is used by all of the plugins in OctoEverywhere! -plugin_version = "3.5.3" +plugin_version = "3.5.4" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From df585922ef7393ab059f7565c5c44565a3fa1115 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sat, 17 Aug 2024 13:45:53 -0700 Subject: [PATCH 144/328] Quick fix for OctoPrint --- octoeverywhere/{notifications => Notifications}/__init__.py | 0 .../{notifications => Notifications}/bedcooldownwatcher.py | 0 octoeverywhere/notificationshandler.py | 2 +- setup.py | 4 ++-- 4 files changed, 3 insertions(+), 3 deletions(-) rename octoeverywhere/{notifications => Notifications}/__init__.py (100%) rename octoeverywhere/{notifications => Notifications}/bedcooldownwatcher.py (100%) diff --git a/octoeverywhere/notifications/__init__.py b/octoeverywhere/Notifications/__init__.py similarity index 100% rename from octoeverywhere/notifications/__init__.py rename to octoeverywhere/Notifications/__init__.py diff --git a/octoeverywhere/notifications/bedcooldownwatcher.py b/octoeverywhere/Notifications/bedcooldownwatcher.py similarity index 100% rename from octoeverywhere/notifications/bedcooldownwatcher.py rename to octoeverywhere/Notifications/bedcooldownwatcher.py diff --git a/octoeverywhere/notificationshandler.py b/octoeverywhere/notificationshandler.py index 6d6da19..da95772 100644 --- a/octoeverywhere/notificationshandler.py +++ b/octoeverywhere/notificationshandler.py @@ -16,7 +16,7 @@ from .printinfo import PrintInfoManager, PrintInfo from .snapshotresizeparams import SnapshotResizeParams from .debugprofiler import DebugProfiler, DebugProfilerFeatures -from .notifications.bedcooldownwatcher import BedCooldownWatcher +from .Notifications.bedcooldownwatcher import BedCooldownWatcher try: # On some systems this package will install but the import will fail due to a missing system .so. diff --git a/setup.py b/setup.py index a37093f..7b49c19 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this single version string is used by all of the plugins in OctoEverywhere! -plugin_version = "3.5.4" +plugin_version = "3.5.6" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module @@ -103,7 +103,7 @@ # Any additional python packages you need to install with your plugin that are not contained in .* # For OctoEverywhere, we need to include or common packages shared between hosts, so OctoPrint copies them into the package folder as well. -plugin_additional_packages = [ "octoeverywhere", "octoeverywhere.Proto", "octoeverywhere.WebStream", "octoeverywhere.Webcam" ] +plugin_additional_packages = [ "octoeverywhere", "octoeverywhere.Proto", "octoeverywhere.WebStream", "octoeverywhere.Webcam", "octoeverywhere.Notifications" ] # Any python packages within .* you do NOT want to install with your plugin plugin_ignored_packages = [] From c7a1d351822e0899257b785061d380b3bda1fd6d Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Thu, 22 Aug 2024 10:45:35 -0700 Subject: [PATCH 145/328] Minor bug fixes --- octoeverywhere/mdns.py | 6 ++++-- octoeverywhere/octohttprequest.py | 18 +++++++++++++----- setup.py | 2 +- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/octoeverywhere/mdns.py b/octoeverywhere/mdns.py index c36d06e..8ada10d 100644 --- a/octoeverywhere/mdns.py +++ b/octoeverywhere/mdns.py @@ -85,8 +85,10 @@ def TryToResolveIfLocalHostnameFound(self, url): hostname = url[protocolEnd:hostnameEnd] self.LogDebug("Found hostname "+hostname+" in url "+url) - # Check if there is a .local hostname. Anything else we will ignore. - if ".local" not in hostname.lower(): + # Check if the hostname ends with .local, which is a special domain that we can resolve. + # We can't do a string contains, because there can be DNS names like "something.local.hostname.com" + hostnameLower = hostname.lower() + if hostnameLower.endswith(".local") or hostnameLower.endswith(".internal"): self.LogDebug("No local domain found in "+url) return None diff --git a/octoeverywhere/octohttprequest.py b/octoeverywhere/octohttprequest.py index 02155b3..6c3bc9c 100644 --- a/octoeverywhere/octohttprequest.py +++ b/octoeverywhere/octohttprequest.py @@ -465,17 +465,14 @@ def MakeHttpCallAttempt(logger, attemptName, method, url, headers, data, mainRes if response is not None and response.status_code != 404: # We got a valid response, we are done. # Return true and the result object, so it can be returned. - return OctoHttpRequest.AttemptResult(True, OctoHttpRequest.Result(response.status_code, response.headers, url, isFallback, requestLibResponseObj=response)) + return OctoHttpRequest.AttemptResult(True, OctoHttpRequest._buildHttRequestResultFromResponse(response, url, isFallback)) # Check if we have another fallback URL to try. if nextFallbackUrl is not None: # We have more fallbacks to try. # Return false so we keep going, but also return this response if we had one. This lets # use capture the main result object, so we can use it eventually if all fallbacks fail. - result = None - if response is not None: - result = OctoHttpRequest.Result(response.status_code, response.headers, url, isFallback, requestLibResponseObj=response) - return OctoHttpRequest.AttemptResult(False, result) + return OctoHttpRequest.AttemptResult(False, OctoHttpRequest._buildHttRequestResultFromResponse(response, url, isFallback)) # We don't have another fallback, so we need to end this. if mainResult is not None: @@ -483,6 +480,17 @@ def MakeHttpCallAttempt(logger, attemptName, method, url, headers, data, mainRes logger.info(attemptName + " failed and we have no more fallbacks. Returning the main URL response.") return OctoHttpRequest.AttemptResult(True, mainResult) else: + if response is not None: + logger.error(attemptName + " failed and we have no more fallbacks. We DON'T have a main response.") + return OctoHttpRequest.AttemptResult(True, OctoHttpRequest._buildHttRequestResultFromResponse(response, url, isFallback)) + # Otherwise return the failure. logger.error(attemptName + " failed and we have no more fallbacks. We DON'T have a main response.") return OctoHttpRequest.AttemptResult(True, None) + + + @staticmethod + def _buildHttRequestResultFromResponse(response:requests.Response, url:str, isFallback:bool) -> Result: + if response is None: + return None + return OctoHttpRequest.Result(response.status_code, response.headers, url, isFallback, requestLibResponseObj=response) diff --git a/setup.py b/setup.py index 7b49c19..c3d6cc6 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this single version string is used by all of the plugins in OctoEverywhere! -plugin_version = "3.5.6" +plugin_version = "3.5.7" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 7bc7163f47b8f098a8b06cac90166e9562052eeb Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Thu, 22 Aug 2024 22:33:01 -0700 Subject: [PATCH 146/328] Minor bug fixes. --- .github/workflows/pylint.yml | 2 +- octoeverywhere/mdns.py | 4 ++-- setup.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 3f28415..187faf9 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -28,7 +28,7 @@ jobs: pip install -r requirements.txt pip install "zstandard>=0.21.0,<0.23.0" - - name: Analysing the code with pylint + - name: Analyzing the code with PYLint run: | pylint ./octoeverywhere/ pylint ./octoprint_octoeverywhere/ diff --git a/octoeverywhere/mdns.py b/octoeverywhere/mdns.py index 8ada10d..aa3e3b4 100644 --- a/octoeverywhere/mdns.py +++ b/octoeverywhere/mdns.py @@ -88,7 +88,7 @@ def TryToResolveIfLocalHostnameFound(self, url): # Check if the hostname ends with .local, which is a special domain that we can resolve. # We can't do a string contains, because there can be DNS names like "something.local.hostname.com" hostnameLower = hostname.lower() - if hostnameLower.endswith(".local") or hostnameLower.endswith(".internal"): + if hostnameLower.endswith(".local") is False and hostnameLower.endswith(".internal") is False: self.LogDebug("No local domain found in "+url) return None @@ -175,7 +175,7 @@ def _TryToResolve(self, domain): # Since we do caching, we allow the lifetime of the lookup to be longer, so we have a better chance of getting it. # Don't allow this to throw, so we don't get nosy exceptions on lookup failures. - answers = self.dnsResolver.resolve(domain, lifetime=3.0, raise_on_no_answer=False, source=localAdapterIp) + answers = self.dnsResolver.resolve(domain, lifetime=1.0, raise_on_no_answer=False, source=localAdapterIp) # Look get the list of IPs returned from the query. Sometimes, there's a multiples. For example, we have seen if docker is installed # there are sometimes 172.x addresses. diff --git a/setup.py b/setup.py index c3d6cc6..7fb38a3 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this single version string is used by all of the plugins in OctoEverywhere! -plugin_version = "3.5.7" +plugin_version = "3.5.8" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From ef3ea059a0956b217d5d7dbb5dd6f2316ac7ea13 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Fri, 30 Aug 2024 21:50:50 -0700 Subject: [PATCH 147/328] Disable errors killing the sh script. --- install.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 4b54ded..f73c4ae 100755 --- a/install.sh +++ b/install.sh @@ -30,7 +30,8 @@ # # Set this to terminate on error. -set -e +# We don't do this anymore, because some commands return non-zero exit codes, but still are successful. +# set -e # From 0be5d91ff75862c6a4db0861f4f0627224ef3ec2 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Mon, 2 Sep 2024 15:40:54 -0700 Subject: [PATCH 148/328] Adding more states for Bambu printers to show. --- bambu_octoeverywhere/bambucloud.py | 5 +- bambu_octoeverywhere/bambucommandhandler.py | 57 ++++++++++++++++++--- 2 files changed, 53 insertions(+), 9 deletions(-) diff --git a/bambu_octoeverywhere/bambucloud.py b/bambu_octoeverywhere/bambucloud.py index fac6356..461504b 100644 --- a/bambu_octoeverywhere/bambucloud.py +++ b/bambu_octoeverywhere/bambucloud.py @@ -209,8 +209,9 @@ def SyncBambuCloudInfo(self) -> bool: if accessCode is None: self.Logger.error("Bambu Cloud SyncBambuCloudInfo didn't find an access code.") return False - # Update the access code if it's changed. - if self.Config.GetStr(Config.SectionBambu, Config.BambuAccessToken, "") != accessCode: + # It turns out that sometimes the Access Code from the service wrong, so we only update + # it if there's no access token set, so the user can override it in the config. + if self.Config.GetStr(Config.SectionBambu, Config.BambuAccessToken, None) is None: self.Config.SetStr(Config.SectionBambu, Config.BambuAccessToken, accessCode) self.Logger.info("Bambu Cloud SyncBambuCloudInfo updated the access code.") return True diff --git a/bambu_octoeverywhere/bambucommandhandler.py b/bambu_octoeverywhere/bambucommandhandler.py index 94e9c0d..96c2a8a 100644 --- a/bambu_octoeverywhere/bambucommandhandler.py +++ b/bambu_octoeverywhere/bambucommandhandler.py @@ -11,6 +11,49 @@ def __init__(self, logger) -> None: self.Logger = logger + # This map contains UI ready strings that map to a subset of sub-stages we can send which are more specific than the state. + # These need to be UI ready, since they will be shown directly. + # Some known stages are excluded, because we don't want to show them. + # Here's a full list: https://github.com/davglass/bambu-cli/blob/398c24057c71fc6bcc5dbd818bdcacc20833f61c/lib/const.js#L104 + SubStageMap = { + 1: "Auto Bed Leveling", + 2: "Bed Preheating", + 3: "Sweeping XY Mech Mode", + 4: "Changing Filament", + 5: "M400 Pause", + 6: "Filament Runout", + 7: "Heating Hotend", + 8: "Calibrating Extrusion", + 9: "Scanning Bed Surface", + 10: "Inspecting First Layer", + 11: "Identifying Build Plate", + 12: "Calibrating Micro Lidar", + 13: "Homing Toolhead", + 14: "Cleaning Nozzle", + 15: "Checking Temperature", + 16: "Paused By User", + 17: "Front Cover Falling", + 18: "Calibrating Micro Lidar", + 19: "Calibrating Extrusion Flow", + 20: "Nozzle Temperature Malfunction", + 21: "Bed Temperature Malfunction", + 22: "Filament Unloading", + 23: "Skip Step Pause", + 24: "Filament Loading", + 25: "Motor Noise Calibration", + 26: "AMS lost", + 27: "Low Speed Of Heat Break Fan", + 28: "Chamber Temperature Control Error", + 29: "Cooling Chamber", + 30: "Paused By Gcode", + 31: "Motor Noise Showoff", + 32: "Nozzle Filament Covered Detected Pause", + 33: "Cutter Error", + 34: "First Layer Error", + 35: "Nozzle Clogged" + } + + # !! Platform Command Handler Interface Function !! # # This must return the common "JobStatus" dict or None on failure. @@ -52,16 +95,10 @@ def GetCurrentJobStatus(self): elif gcodeState == "RUNNING" or gcodeState == "SLICING": # Only check stg_cur in the known printing state, because sometimes it doesn't get reset to idle when transitioning to an error. stg = bambuState.stg_cur - # Here's a full list: https://github.com/davglass/bambu-cli/blob/398c24057c71fc6bcc5dbd818bdcacc20833f61c/lib/const.js#L104 - # stg==255 is used as a kind of intenum unknown state when the print is first starting and finishing. - # We can't really use it because it can happen at different points in time and it's not clear what the real state is. if stg == 2 or stg == 7: state = "warmingup" - elif stg == 14: - state = "cleaningnozzle" - elif stg == 1: - state = "autobedlevel" else: + # These are all a subset of printing states. state = "printing" elif gcodeState == "PAUSE": state = "paused" @@ -79,6 +116,11 @@ def GetCurrentJobStatus(self): else: self.Logger.warn(f"Unknown gcode_state state in print state: {gcodeState}") + # If we have a mapped sub state, set it. + subState_CanBeNone = None + if bambuState.stg_cur is not None: + if bambuState.stg_cur in BambuCommandHandler.SubStageMap: + subState_CanBeNone = BambuCommandHandler.SubStageMap[bambuState.stg_cur] # Get current layer info # None = The platform doesn't provide it. @@ -137,6 +179,7 @@ def GetCurrentJobStatus(self): # Build the object and return. return { "State": state, + "SubState": subState_CanBeNone, "Error": errorStr_CanBeNone, "CurrentPrint": { From ecef778fbea6eacdf3232de934b86bf6d8dd92d5 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Mon, 2 Sep 2024 20:49:53 -0700 Subject: [PATCH 149/328] Version bump! --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7fb38a3..c93b3c2 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this single version string is used by all of the plugins in OctoEverywhere! -plugin_version = "3.5.8" +plugin_version = "3.5.9" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 4d4b053c4745431c53dd82b9a51abb1e5797fd60 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Wed, 4 Sep 2024 17:56:29 -0700 Subject: [PATCH 150/328] Major CPU and memory improments for webcam streaming! --- .../WebStream/octowebstreamhttphelper.py | 296 ++++++++++-------- octoeverywhere/compression.py | 4 + octoeverywhere/octostreammsgbuilder.py | 2 +- requirements.txt | 2 +- setup.py | 4 +- 5 files changed, 179 insertions(+), 129 deletions(-) diff --git a/octoeverywhere/WebStream/octowebstreamhttphelper.py b/octoeverywhere/WebStream/octowebstreamhttphelper.py index c7cba14..b441d93 100644 --- a/octoeverywhere/WebStream/octowebstreamhttphelper.py +++ b/octoeverywhere/WebStream/octowebstreamhttphelper.py @@ -5,6 +5,7 @@ import requests import urllib3 +import octoflatbuffers from .octoheaderimpl import HeaderHelper from .octoheaderimpl import BaseProtocol @@ -23,6 +24,22 @@ from ..Proto import OeAuthAllowed from ..Proto.PathTypes import PathTypes +# A wrapper that allows us to pass around a ref to the per message builder object. +class MsgBuilderContext: + + # From testing, beyond the dynamic size of the send buffer, the rest of the overhead is from 500-2000 bytes. + # This overhead does include some dynamic things, like the return headers, and other random values. + # It's way better to be over the size than under, since if we are under the buffer will resize, double in size, and do a copy. + # Thus, we allocate 10k bytes for the overhead, which should be more than enough. + c_MsgStreamOverheadSize = 1024 * 10 + + def __init__(self): + self.Builder:octoflatbuffers.Builder = None + + def CreateBuilder(self, knownBodySizeBytes = 0): + self.Builder = octoflatbuffers.Builder(knownBodySizeBytes + self.c_MsgStreamOverheadSize) + + # # A helper object that handles http request for the web stream system. # @@ -42,7 +59,7 @@ def __init__(self, streamId, logger:logging.Logger, webStream, webStreamOpenMsg, self.CompressionContext = CompressionContext(self.Logger) # Vars for response reading - self.BodyReadTempBuffer = None + self.BodyReadTempBuffer:bytearray = None self.ChunkedBodyHasNoContentLengthHeaders = False self.CompressionType:DataCompression.DataCompression = None self.CompressionTimeSec = -1 @@ -330,8 +347,9 @@ def executeHttpRequest(self): self.Logger.info(f"We detected that the compression being applied to this stream was inefficient, so we are disabling compression. Compression: {float(contentReadBytes)/float(nonCompressedContentReadSizeBytes)} URL: {uri}") # Prepare a response. - # TODO - We should start the buffer at something that's likely to not need expanding for most requests. - builder = OctoStreamMsgBuilder.CreateBuffer(20000) + # In the past we started the message here, but the problem is we don't really know how large to make it. + # So instead, we build this context and let the body read function tell us how much data it read. + builderContext = MsgBuilderContext() # Unless we are skipping the body read, do it now. # If there's a 304, we might have a body, but we don't want to read it. @@ -346,10 +364,14 @@ def executeHttpRequest(self): # Start by reading data from the response. # This function will return a read length of 0 and a null data offset if there's nothing to read. # Otherwise, it will return the length of the read data and the data offset in the buffer. - nonCompressedBodyReadSize, lastBodyReadLength, dataOffset = self.readContentFromBodyAndMakeDataVector(builder, octoHttpResult, boundaryStr, compressBody, contentTypeLower, contentLength, responseHandlerContext) + nonCompressedBodyReadSize, lastBodyReadLength, dataOffset = self.readContentFromBodyAndMakeDataVector(builderContext, octoHttpResult, boundaryStr, compressBody, contentTypeLower, contentLength, responseHandlerContext) contentReadBytes += lastBodyReadLength nonCompressedContentReadSizeBytes += nonCompressedBodyReadSize + # Ensure that the build was created by now. In most cases it's created with the body read, but in other cases where there's no body, we create it now. + if builderContext.Builder is None: + builderContext.CreateBuilder() + # Special Case - If this request was handled by the Web Request Response Handler, the body buffer might have been edited. # We need to update the content length for the message, so it's sent correctly in the OctoStream response. # Since we know we read the entire file at once, this should be the first message, which means updating it now @@ -385,41 +407,41 @@ def executeHttpRequest(self): statusCode = octoHttpResult.StatusCode # Gather the headers, if there are any. This will return None if there are no headers to send. - headerVectorOffset = self.buildHeaderVector(builder, octoHttpResult) + headerVectorOffset = self.buildHeaderVector(builderContext.Builder, octoHttpResult) # Build the initial context. We should always send a http initial context on the first response, # even if there are no headers in t. - HttpInitialContext.Start(builder) + HttpInitialContext.Start(builderContext.Builder) if headerVectorOffset is not None: - HttpInitialContext.AddHeaders(builder, headerVectorOffset) - httpInitialContextOffset = HttpInitialContext.End(builder) + HttpInitialContext.AddHeaders(builderContext.Builder, headerVectorOffset) + httpInitialContextOffset = HttpInitialContext.End(builderContext.Builder) # Now build the return message - WebStreamMsg.Start(builder) - WebStreamMsg.AddStreamId(builder, self.Id) + WebStreamMsg.Start(builderContext.Builder) + WebStreamMsg.AddStreamId(builderContext.Builder, self.Id) # Indicate this message has data, even if it's just the initial http context (because there's no data for this request) - WebStreamMsg.AddIsControlFlagsOnly(builder, False) + WebStreamMsg.AddIsControlFlagsOnly(builderContext.Builder, False) if statusCode is not None: - WebStreamMsg.AddStatusCode(builder, statusCode) + WebStreamMsg.AddStatusCode(builderContext.Builder, statusCode) if dataOffset is not None: - WebStreamMsg.AddData(builder, dataOffset) + WebStreamMsg.AddData(builderContext.Builder, dataOffset) if httpInitialContextOffset is not None: # This should always be not null for the first response. - WebStreamMsg.AddHttpInitialContext(builder, httpInitialContextOffset) + WebStreamMsg.AddHttpInitialContext(builderContext.Builder, httpInitialContextOffset) if isFirstResponse is True and contentLength is not None: # Only on the first response, if we know the full size, set it. - WebStreamMsg.AddFullStreamDataSize(builder, contentLength) + WebStreamMsg.AddFullStreamDataSize(builderContext.Builder, contentLength) if compressBody: # If we are compressing, we need to add what we are using and what the original size was. if self.CompressionType is None: raise Exception("The body of this message should be compressed but not compression type is set.") - WebStreamMsg.AddDataCompression(builder, self.CompressionType) - WebStreamMsg.AddOriginalDataSize(builder, nonCompressedBodyReadSize) + WebStreamMsg.AddDataCompression(builderContext.Builder, self.CompressionType) + WebStreamMsg.AddOriginalDataSize(builderContext.Builder, nonCompressedBodyReadSize) if isLastMessage: # If this is the last message because we know the body is all # sent, indicate that the data stream is done and send the close message. - WebStreamMsg.AddIsDataTransmissionDone(builder, True) - WebStreamMsg.AddIsCloseMsg(builder, True) + WebStreamMsg.AddIsDataTransmissionDone(builderContext.Builder, True) + WebStreamMsg.AddIsCloseMsg(builderContext.Builder, True) if self.MultipartReadsPerSecond != 0: # If this is a multipart stream (webcam streaming), every 1 second a value will be dumped into MultipartReadsPerSecond # when it's there, we want to send it to the server for telemetry, and then zero it out. @@ -428,25 +450,25 @@ def executeHttpRequest(self): if self.MultipartReadsPerSecond > 255 or self.MultipartReadsPerSecond < 0: self.Logger.warn("self.MultipartReadsPerSecond is larger than uint8. "+str(self.MultipartReadsPerSecond)) self.MultipartReadsPerSecond = 255 - WebStreamMsg.AddMultipartReadsPerSecond(builder, self.MultipartReadsPerSecond) + WebStreamMsg.AddMultipartReadsPerSecond(builderContext.Builder, self.MultipartReadsPerSecond) self.MultipartReadsPerSecond = 0 # Also attach the other stats. bodyReadTimeHighWaterMarkMs = int(self.BodyReadTimeHighWaterMarkSec * 1000.0) self.BodyReadTimeHighWaterMarkSec = 0.0 if bodyReadTimeHighWaterMarkMs > 65535 or bodyReadTimeHighWaterMarkMs < 0: bodyReadTimeHighWaterMarkMs = 65535 - WebStreamMsg.AddBodyReadTimeHighWaterMarkMs(builder, bodyReadTimeHighWaterMarkMs) + WebStreamMsg.AddBodyReadTimeHighWaterMarkMs(builderContext.Builder, bodyReadTimeHighWaterMarkMs) serviceUploadTimeHighWaterMarkMs = int(self.ServiceUploadTimeHighWaterMarkSec * 1000.0) self.ServiceUploadTimeHighWaterMarkSec = 0.0 if serviceUploadTimeHighWaterMarkMs > 65535 or serviceUploadTimeHighWaterMarkMs < 0: serviceUploadTimeHighWaterMarkMs = 65535 - WebStreamMsg.AddSocketSendTimeHighWaterMarkMs(builder, serviceUploadTimeHighWaterMarkMs) + WebStreamMsg.AddSocketSendTimeHighWaterMarkMs(builderContext.Builder, serviceUploadTimeHighWaterMarkMs) - webStreamMsgOffset = WebStreamMsg.End(builder) + webStreamMsgOffset = WebStreamMsg.End(builderContext.Builder) # Wrap in the OctoStreamMsg and finalize. - outputBuf = OctoStreamMsgBuilder.CreateOctoStreamMsgAndFinalize(builder, MessageContext.MessageContext.WebStreamMsg, webStreamMsgOffset) + outputBuf = OctoStreamMsgBuilder.CreateOctoStreamMsgAndFinalize(builderContext.Builder, MessageContext.MessageContext.WebStreamMsg, webStreamMsgOffset) # Send the message. # If this is the last, we need to make sure to set that we have set the closed flag. @@ -457,6 +479,13 @@ def executeHttpRequest(self): if thisServiceSendTimeSec > self.ServiceUploadTimeHighWaterMarkSec: self.ServiceUploadTimeHighWaterMarkSec = thisServiceSendTimeSec + # Do a debug check to see if our pre-allocated flatbuffer size was too small. + # If this fires often, we should increase the c_MsgStreamOverheadSize size. + finalFullBufferBytes = len(builderContext.Builder.Bytes) + if finalFullBufferBytes > lastBodyReadLength + builderContext.c_MsgStreamOverheadSize and self.Logger.isEnabledFor(logging.DEBUG): + delta = len(outputBuf) - (lastBodyReadLength + builderContext.c_MsgStreamOverheadSize) + self.Logger.warn(f"The flatbuffer internal buffer had to be resized from the guess we set. Flatbuffer full buffer size: {finalFullBufferBytes}, last body read length: {lastBodyReadLength}; overrage delta: {delta}") + # Clear this flag isFirstResponse = False messageCount += 1 @@ -516,6 +545,7 @@ def finalizeUnknownUploadSizeIfNeeded(self): # match how much we have, that's an error that will be thrown later. if self.UploadBuffer is not None and self.KnownFullStreamUploadSizeBytes is None: # Trim the buffer to the final size that we received. + # This will do a copy and set the copy as the upload buffer. self.UploadBuffer = self.UploadBuffer[0:self.UploadBytesReceivedSoFar] @@ -535,9 +565,6 @@ def copyUploadDataFromMsg(self, webStreamMsg:WebStreamMsg.WebStreamMsg): # just use this buffer. if self.UploadBuffer is None and self.KnownFullStreamUploadSizeBytes is not None and self.KnownFullStreamUploadSizeBytes == thisMessageDataLen: # This is the only message with data, just use it's buffer. - # I -believe- this doesn't copy the buffer and just makes a view of it. - # That's the ideal case, because this message buffer will stay around since - # the http will execute on this same stack. self.UploadBuffer = self.decompressBufferIfNeeded(webStreamMsg) self.UploadBytesReceivedSoFar = len(self.UploadBuffer) # Done! @@ -586,13 +613,14 @@ def copyUploadDataFromMsg(self, webStreamMsg:WebStreamMsg.WebStreamMsg): # A helper, given a web stream message returns it's data buffer, decompressed if needed. - def decompressBufferIfNeeded(self, webStreamMsg:WebStreamMsg.WebStreamMsg): + def decompressBufferIfNeeded(self, webStreamMsg:WebStreamMsg.WebStreamMsg) -> bytearray: # Get the compression type. compressionType = webStreamMsg.DataCompression() + dataByteArray = webStreamMsg.DataAsByteArray() if compressionType is DataCompression.DataCompression.None_: - return webStreamMsg.DataAsByteArray() + return dataByteArray # It's compressed, decompress it. - return Compression.Get().Decompress(self.CompressionContext, webStreamMsg.DataAsByteArray(), webStreamMsg.OriginalDataSize(), webStreamMsg.IsDataTransmissionDone(), compressionType) + return Compression.Get().Decompress(self.CompressionContext, dataByteArray, webStreamMsg.OriginalDataSize(), webStreamMsg.IsDataTransmissionDone(), compressionType) def checkForNotModifiedCacheAndUpdateResponseIfSo(self, sentHeaders, octoHttpResult:OctoHttpRequest.Result): @@ -712,7 +740,7 @@ def shouldCompressBody(self, contentTypeLower:str, octoHttpResult:OctoHttpReques # Reads data from the response body, puts it in a data vector, and returns the offset. # If the body has been fully read, this should return ogLen == 0, len = 0, and offset == None # The read style depends on the presence of the boundary string existing. - def readContentFromBodyAndMakeDataVector(self, builder, octoHttpResult:OctoHttpRequest.Result, boundaryStr_opt, shouldCompress, contentTypeLower_NoneIfNotKnown, contentLength_NoneIfNotKnown, responseHandlerContext): + def readContentFromBodyAndMakeDataVector(self, builderContext:MsgBuilderContext, octoHttpResult:OctoHttpRequest.Result, boundaryStr_opt, shouldCompress, contentTypeLower_NoneIfNotKnown, contentLength_NoneIfNotKnown, responseHandlerContext): # This is the max size each body read will be. Since we are making local calls, most of the time we will always get this full amount as long as theres more body to read. # This size is a little under the max read buffer on the server, allowing the server to handle the buffers with no copies. # @@ -738,102 +766,120 @@ def readContentFromBodyAndMakeDataVector(self, builder, octoHttpResult:OctoHttpR # Some requests like snapshot requests will already have a fully read body. In this case we use the existing body buffer instead of reading from the body. finalDataBuffer = None - bodyReadStartSec = time.time() - if self.IsUsingFullBodyBuffer: - # In this case, the entire buffer and size are known, so we get them all in one go. - finalDataBuffer = octoHttpResult.FullBodyBuffer - elif self.IsUsingCustomBodyStreamCallbacks: - # In this case we just call this callback, and send whatever it sends. Note that even if this is a boundary stream, we just send back what it sends. - # If None is returned, we are done. - finalDataBuffer = octoHttpResult.GetCustomBodyStreamCallback() - else: - # If the boundary string exist and is not empty, we will use it to try to read the data. - # Unless the self.ChunkedBodyHasNoContentLengthHeaders flag has been set, which indicate we have read the body has chunks - # and failed to find any content length headers. In that case, we will just read fixed sized chunks. - if self.ChunkedBodyHasNoContentLengthHeaders is False and boundaryStr_opt is not None and len(boundaryStr_opt) != 0: - # Try to read a single boundary chunk - readLength = self.readStreamChunk(octoHttpResult, boundaryStr_opt) - # If we get a length, set the final buffer using the temp buffer. - # This isn't a copy, just a reference to a subset of the buffer. - if readLength != 0: - finalDataBuffer = self.BodyReadTempBuffer[0:readLength] + finalDataBufferMv_CanBeNone = None + try: + bodyReadStartSec = time.time() + if self.IsUsingFullBodyBuffer: + # In this case, the entire buffer and size are known, so we get them all in one go. + finalDataBuffer = octoHttpResult.FullBodyBuffer + elif self.IsUsingCustomBodyStreamCallbacks: + # In this case we just call this callback, and send whatever it sends. Note that even if this is a boundary stream, we just send back what it sends. + # If None is returned, we are done. + finalDataBuffer = octoHttpResult.GetCustomBodyStreamCallback() else: - if responseHandlerContext is None and self.shouldDoUnknownBodySizeRead(contentTypeLower_NoneIfNotKnown, contentLength_NoneIfNotKnown): - # If we don't know the content length AND there is no boundary string, this request is probably a event stream of some sort. - # We have to use this special read function, because doBodyRead will block until the full buffer is filled, which might take a long time - # for a number of streamed messages to fill it up. This special function does micro reads on the socket until a time limit is hit, and then - # returns what was received. - self.IsDoingMicroBodyReads = True - finalDataBuffer = self.doUnknownBodySizeRead(octoHttpResult) + # If the boundary string exist and is not empty, we will use it to try to read the data. + # Unless the self.ChunkedBodyHasNoContentLengthHeaders flag has been set, which indicate we have read the body has chunks + # and failed to find any content length headers. In that case, we will just read fixed sized chunks. + if self.ChunkedBodyHasNoContentLengthHeaders is False and boundaryStr_opt is not None and len(boundaryStr_opt) != 0: + # Try to read a single boundary chunk + readLength = self.readStreamChunk(octoHttpResult, boundaryStr_opt) + # If we get a length, we have a buffer to use. + if readLength != 0: + # We create a memory view from the buffer, which is a zero copy operation and zero copy slicing. + # This allows us to pass the buffer around without copying it, but we do have to be sure to release the + # memory views when we are done. + finalDataBufferMv_CanBeNone = memoryview(self.BodyReadTempBuffer) + finalDataBuffer = finalDataBufferMv_CanBeNone[0:readLength] else: - # If there is no boundary string, but we know the content length, it's safe to just read. - # This will block until either the full defaultBodyReadSizeBytes is read or the full request has been received. - # If this returns None, we hit a read timeout or the stream is done, so we are done. - - # If this request will be handled by the a response handler, we need to load the full body into one buffer. - if responseHandlerContext: - # We have to be careful with the size, because on some platforms (like the K1) whatever size we pass it will try to allocate - # into one buffer. If we know the context length, us it. Otherwise, set something that's reasonably large. - if contentLength_NoneIfNotKnown is not None: - defaultBodyReadSizeBytes = contentLength_NoneIfNotKnown - else: - # Use a 2mb buffer. - defaultBodyReadSizeBytes = 1024 * 1024 * 2 - finalDataBuffer = self.doBodyRead(octoHttpResult, defaultBodyReadSizeBytes) - - # Keep track of read times. - thisBodyReadTimeSec = time.time() - bodyReadStartSec - self.BodyReadTimeSec += thisBodyReadTimeSec - if thisBodyReadTimeSec > self.BodyReadTimeHighWaterMarkSec: - self.BodyReadTimeHighWaterMarkSec = thisBodyReadTimeSec - - # If the final data buffer has been set to None, it means the body is not empty - if finalDataBuffer is None: - # Return empty to indicate the body has been fully read. - return (0, 0, None) - - # Before we do any compression, check if there is a response handler context, meaning there's a response handler that - # might want to edit the body buffer before it's compressed. - if responseHandlerContext: - if contentLength_NoneIfNotKnown is not None and len(finalDataBuffer) != contentLength_NoneIfNotKnown: - self.Logger.error("We detected the read of the web request response handler message, but the buffer size doesn't match the content length.") - else: - # If we have the compat handler, give it the buffer before we finalize the size, as it might want to edit the buffer. - if Compat.HasWebRequestResponseHandler(): - finalDataBuffer = Compat.GetWebRequestResponseHandler().HandleResponse(responseHandlerContext, octoHttpResult, finalDataBuffer) - # Important! If the response handler has edited the buffer, we need to update the content length to match the new size. - # This is safe to do because currently we always read the entire buffer for a responseHandlerContext into one buffer, thus there's only one read, and this is the read. - # The function that calls readContentFromBodyAndMakeDataVector will correct the content header length in the main class, but we must update the encryption context - # otherwise the zstandard lib encryption will fail. - self.CompressionContext.SetTotalCompressedSizeOfData(len(finalDataBuffer)) - - # If we were asked to compress, do it - originalBufferSize = len(finalDataBuffer) - - # Check to see if this was a full body buffer, if it was already compressed. - if octoHttpResult.BodyBufferCompressionType != DataCompression.DataCompression.None_: - # The full body buffer was already compressed and set, so update the other compression values. - originalBufferSize = octoHttpResult.BodyBufferPreCompressSize - if self.CompressionType is not None: - raise Exception(f"The BodyBufferCompressionType tried to be set but the compression was already set! It is {self.CompressionType} and now tried to be {octoHttpResult.BodyBufferCompressionType}") - self.CompressionType = octoHttpResult.BodyBufferCompressionType - - # Otherwise, check if we should compress - elif shouldCompress: - compressionResult = Compression.Get().Compress(self.CompressionContext, finalDataBuffer) - finalDataBuffer = compressionResult.Bytes - # Init and update the total compression time if needed. - if self.CompressionTimeSec < 0: - self.CompressionTimeSec = 0 - self.CompressionTimeSec += compressionResult.CompressionTimeSec - # Set the compression type, this should only be set once and can't change. - if self.CompressionType is None: - self.CompressionType = compressionResult.CompressionType - elif self.CompressionType != compressionResult.CompressionType: - raise Exception(f"The data compression has changed mid stream! It was {self.CompressionType} and now tried to be {compressionResult.CompressionType}") - - # We have a data buffer, so write it into the builder and return the offset. - return (originalBufferSize, len(finalDataBuffer), builder.CreateByteVector(finalDataBuffer)) + if responseHandlerContext is None and self.shouldDoUnknownBodySizeRead(contentTypeLower_NoneIfNotKnown, contentLength_NoneIfNotKnown): + # If we don't know the content length AND there is no boundary string, this request is probably a event stream of some sort. + # We have to use this special read function, because doBodyRead will block until the full buffer is filled, which might take a long time + # for a number of streamed messages to fill it up. This special function does micro reads on the socket until a time limit is hit, and then + # returns what was received. + self.IsDoingMicroBodyReads = True + finalDataBuffer = self.doUnknownBodySizeRead(octoHttpResult) + else: + # If there is no boundary string, but we know the content length, it's safe to just read. + # This will block until either the full defaultBodyReadSizeBytes is read or the full request has been received. + # If this returns None, we hit a read timeout or the stream is done, so we are done. + + # If this request will be handled by the a response handler, we need to load the full body into one buffer. + if responseHandlerContext: + # We have to be careful with the size, because on some platforms (like the K1) whatever size we pass it will try to allocate + # into one buffer. If we know the context length, us it. Otherwise, set something that's reasonably large. + if contentLength_NoneIfNotKnown is not None: + defaultBodyReadSizeBytes = contentLength_NoneIfNotKnown + else: + # Use a 2mb buffer. + defaultBodyReadSizeBytes = 1024 * 1024 * 2 + finalDataBuffer = self.doBodyRead(octoHttpResult, defaultBodyReadSizeBytes) + + # Keep track of read times. + thisBodyReadTimeSec = time.time() - bodyReadStartSec + self.BodyReadTimeSec += thisBodyReadTimeSec + if thisBodyReadTimeSec > self.BodyReadTimeHighWaterMarkSec: + self.BodyReadTimeHighWaterMarkSec = thisBodyReadTimeSec + + # If the final data buffer has been set to None, it means the body is not empty + if finalDataBuffer is None: + # Return empty to indicate the body has been fully read. + return (0, 0, None) + + # Before we do any compression, check if there is a response handler context, meaning there's a response handler that + # might want to edit the body buffer before it's compressed. + if responseHandlerContext: + if contentLength_NoneIfNotKnown is not None and len(finalDataBuffer) != contentLength_NoneIfNotKnown: + self.Logger.error("We detected the read of the web request response handler message, but the buffer size doesn't match the content length.") + else: + # If we have the compat handler, give it the buffer before we finalize the size, as it might want to edit the buffer. + if Compat.HasWebRequestResponseHandler(): + finalDataBuffer = Compat.GetWebRequestResponseHandler().HandleResponse(responseHandlerContext, octoHttpResult, finalDataBuffer) + # Important! If the response handler has edited the buffer, we need to update the content length to match the new size. + # This is safe to do because currently we always read the entire buffer for a responseHandlerContext into one buffer, thus there's only one read, and this is the read. + # The function that calls readContentFromBodyAndMakeDataVector will correct the content header length in the main class, but we must update the encryption context + # otherwise the zstandard lib encryption will fail. + self.CompressionContext.SetTotalCompressedSizeOfData(len(finalDataBuffer)) + + # If we were asked to compress, do it + originalBufferSize = len(finalDataBuffer) + + # Check to see if this was a full body buffer, if it was already compressed. + if octoHttpResult.BodyBufferCompressionType != DataCompression.DataCompression.None_: + # The full body buffer was already compressed and set, so update the other compression values. + originalBufferSize = octoHttpResult.BodyBufferPreCompressSize + if self.CompressionType is not None: + raise Exception(f"The BodyBufferCompressionType tried to be set but the compression was already set! It is {self.CompressionType} and now tried to be {octoHttpResult.BodyBufferCompressionType}") + self.CompressionType = octoHttpResult.BodyBufferCompressionType + + # Otherwise, check if we should compress + elif shouldCompress: + compressionResult = Compression.Get().Compress(self.CompressionContext, finalDataBuffer) + finalDataBuffer = compressionResult.Bytes + # Init and update the total compression time if needed. + if self.CompressionTimeSec < 0: + self.CompressionTimeSec = 0 + self.CompressionTimeSec += compressionResult.CompressionTimeSec + # Set the compression type, this should only be set once and can't change. + if self.CompressionType is None: + self.CompressionType = compressionResult.CompressionType + elif self.CompressionType != compressionResult.CompressionType: + raise Exception(f"The data compression has changed mid stream! It was {self.CompressionType} and now tried to be {compressionResult.CompressionType}") + + # We have a data buffer and we know how large it will be. + # Since this buffer is the majority of the flatbuffer message, we use it to create the initial size of the flatbuffer. + # This is important, because if the buffer is too small, it will double the size in a loop until it's big enough, which is silly. + # So ideally we use the size of the body buffer we will actually send, and add enough overhead to contain the rest of the msg data. + finalDataBufferSizeBytes = len(finalDataBuffer) + builderContext.CreateBuilder(finalDataBufferSizeBytes) + + return (originalBufferSize, len(finalDataBuffer), builderContext.Builder.CreateByteVector(finalDataBuffer)) + finally: + # If we used a memory view, release it. + # This also means that the finalDataBuffer is a memory view. + if finalDataBufferMv_CanBeNone is not None: + finalDataBuffer.release() + finalDataBufferMv_CanBeNone.release() + # Reads a single chunk from the http response. # This function uses the BodyReadTempBuffer to store the data. diff --git a/octoeverywhere/compression.py b/octoeverywhere/compression.py index bf0734f..dd84879 100644 --- a/octoeverywhere/compression.py +++ b/octoeverywhere/compression.py @@ -71,6 +71,7 @@ def __exit__(self, exc_type, exc_value, traceback): compressor = None streamReader = None decompressor = None + with self.ResourceLock: self.IsClosed = True @@ -106,6 +107,9 @@ def SetTotalCompressedSizeOfData(self, totalSizeBytes:int): # This is the callback from stream_writer that get called when it has data to write. def write(self, data): + # A bytearray is a better option if we are continuously appending data, since we can allocate a bigger buffer + # and copy into it. But 99% of the time we are only doing one compress callback at a time, in which case it's + # better to just take the buffer given to us and use it. if self.CompressionByteBuffer is None: self.CompressionByteBuffer = data else: diff --git a/octoeverywhere/octostreammsgbuilder.py b/octoeverywhere/octostreammsgbuilder.py index 213f3e9..572f452 100644 --- a/octoeverywhere/octostreammsgbuilder.py +++ b/octoeverywhere/octostreammsgbuilder.py @@ -51,7 +51,7 @@ def BuildHandshakeSyn(printerId, privateKey, isPrimarySession, pluginVersion, lo return OctoStreamMsgBuilder.CreateOctoStreamMsgAndFinalize(builder, MessageContext.MessageContext.HandshakeSyn, synOffset) @staticmethod - def CreateBuffer(size): + def CreateBuffer(size) -> octoflatbuffers.Builder: return octoflatbuffers.Builder(size) @staticmethod diff --git a/requirements.txt b/requirements.txt index d02997c..6c67803 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ # websocket_client>=1.6.0,<1.7.99 requests>=2.31.0 -octoflatbuffers==24.3.26 +octoflatbuffers==24.3.27 pillow certifi>=2023.11.17 rsa>=4.9 diff --git a/setup.py b/setup.py index c93b3c2..cf2a290 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this single version string is used by all of the plugins in OctoEverywhere! -plugin_version = "3.5.9" +plugin_version = "3.6.0" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module @@ -80,7 +80,7 @@ plugin_requires = [ "websocket_client>=1.6.0,<1.7.99", "requests>=2.31.0", - "octoflatbuffers==24.3.26", + "octoflatbuffers==24.3.27", "pillow", "certifi>=2023.11.17", "rsa>=4.9", From de6cd726a48c3be4b802346866ebdb981b9ad7d5 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Thu, 5 Sep 2024 18:19:30 -0700 Subject: [PATCH 151/328] Upgrading to our new websocket lib that have major perf benfits! --- .vscode/settings.json | 2 ++ moonraker_octoeverywhere/moonrakerclient.py | 4 +-- .../WebStream/octowebstreamwshelper.py | 12 ++++----- octoeverywhere/websocketimpl.py | 25 +++++++++++++------ requirements.txt | 2 +- setup.py | 15 ++++++----- 6 files changed, 35 insertions(+), 25 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index fdb6093..6138161 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -160,6 +160,7 @@ "octosessionimpl", "octostream", "octostreammsgbuilder", + "octowebsocket", "octowebstream", "octowebstreamhttphelper", "octowebstreamhttphelperimpl", @@ -215,6 +216,7 @@ "rtsps", "sdcard", "serverauth", + "setdefaulttimeout", "shotty", "skipsudoactions", "smartpause", diff --git a/moonraker_octoeverywhere/moonrakerclient.py b/moonraker_octoeverywhere/moonrakerclient.py index 34776fa..c936354 100644 --- a/moonraker_octoeverywhere/moonrakerclient.py +++ b/moonraker_octoeverywhere/moonrakerclient.py @@ -7,7 +7,7 @@ import math import configparser -import websocket +import octowebsocket from octoeverywhere.compat import Compat from octoeverywhere.sentry import Sentry @@ -749,7 +749,7 @@ def _onWsError(self, ws, exception): if Client.IsCommonConnectionException(exception): # Don't bother logging, this just means there's no server to connect to. pass - elif isinstance(exception, websocket.WebSocketBadStatusException) and "Handshake status" in str(exception): + elif isinstance(exception, octowebsocket.WebSocketBadStatusException) and "Handshake status" in str(exception): # This is moonraker specific, we sometimes see stuff like "Handshake status 502 Bad Gateway" self.Logger.info(f"Failed to connect to moonraker due to bad gateway stats. {exception}") else: diff --git a/octoeverywhere/WebStream/octowebstreamwshelper.py b/octoeverywhere/WebStream/octowebstreamwshelper.py index 9e78b47..bbc5c31 100644 --- a/octoeverywhere/WebStream/octowebstreamwshelper.py +++ b/octoeverywhere/WebStream/octowebstreamwshelper.py @@ -3,7 +3,7 @@ import time import threading -import websocket +import octowebsocket from ..mdns import MDns from ..sentry import Sentry @@ -283,11 +283,11 @@ def IncomingServerMessage(self, webStreamMsg:WebStreamMsg.WebStreamMsg): sendType = 0 msgType = webStreamMsg.WebsocketDataType() if msgType == WebSocketDataTypes.WebSocketDataTypes.Text: - sendType = websocket.ABNF.OPCODE_TEXT + sendType = octowebsocket.ABNF.OPCODE_TEXT elif msgType == WebSocketDataTypes.WebSocketDataTypes.Binary: - sendType = websocket.ABNF.OPCODE_BINARY + sendType = octowebsocket.ABNF.OPCODE_BINARY elif msgType == WebSocketDataTypes.WebSocketDataTypes.Close: - sendType = websocket.ABNF.OPCODE_CLOSE + sendType = octowebsocket.ABNF.OPCODE_CLOSE else: raise Exception("Web stream ws was sent a data type that's unknown. "+str(msgType)) @@ -320,9 +320,9 @@ def onWsData(self, ws, buffer:bytes, msgType): # Figure out the data type # TODO - we should support the OPCODE_CONT type at some point. But it's not needed right now. sendType = WebSocketDataTypes.WebSocketDataTypes.None_ - if msgType == websocket.ABNF.OPCODE_BINARY: + if msgType == octowebsocket.ABNF.OPCODE_BINARY: sendType = WebSocketDataTypes.WebSocketDataTypes.Binary - elif msgType == websocket.ABNF.OPCODE_TEXT: + elif msgType == octowebsocket.ABNF.OPCODE_TEXT: sendType = WebSocketDataTypes.WebSocketDataTypes.Text # In PY3 using the modern websocket_client lib the text also comes as a byte buffer. else: diff --git a/octoeverywhere/websocketimpl.py b/octoeverywhere/websocketimpl.py index f68ff0b..05c16f2 100644 --- a/octoeverywhere/websocketimpl.py +++ b/octoeverywhere/websocketimpl.py @@ -1,8 +1,8 @@ import queue import threading import certifi -import websocket -from websocket import WebSocketApp +import octowebsocket +from octowebsocket import WebSocketApp from .sentry import Sentry @@ -11,6 +11,12 @@ class Client: def __init__(self, url, onWsOpen = None, onWsMsg = None, onWsData = None, onWsClose = None, onWsError = None, headers:dict = None, subProtocolList:list = None): + # Set the default timeout for the socket. There's no other way to do this than this global var, and it will be shared by all websockets. + # This is used when the system is writing or receiving, but not when it's waiting to receive, as that's a select() + # We set it to be something high because most all errors will be handled other ways, but this prevents the websocket from hanging forever. + # The value is in seconds, we currently set it to 10 minutes. + octowebsocket.setdefaulttimeout(10 * 60) + # Since we also fire onWsError if there is a send error, we need to capture # the callback and have some vars to ensure it only gets fired once. self.clientWsErrorCallback = onWsError @@ -185,9 +191,9 @@ def fireWsErrorCallbackThread(self, exception): def Send(self, msgBytes, isData): if isData: - self.SendWithOptCode(msgBytes, websocket.ABNF.OPCODE_BINARY) + self.SendWithOptCode(msgBytes, octowebsocket.ABNF.OPCODE_BINARY) else: - self.SendWithOptCode(msgBytes, websocket.ABNF.OPCODE_TEXT) + self.SendWithOptCode(msgBytes, octowebsocket.ABNF.OPCODE_TEXT) def SendWithOptCode(self, msgBytes, opcode): @@ -211,7 +217,10 @@ def _SendQueueThread(self): if context is None or context.Buffer is None: return # Send it! - self.Ws.send(context.Buffer, context.OptCode) + # Important! We don't want to use the frame mask because it adds about 30% CPU usage on low end devices. + # The frame masking was only need back when websockets were used over the internet without SSL. + # Our server, OctoPrint, and Moonraker all accept unmasked frames, so its safe to do this for all WS. + self.Ws.send(context.Buffer, context.OptCode, False) except Exception as e: # If any exception happens during sending, we want to report the error # and shutdown the entire websocket. @@ -259,15 +268,15 @@ def IsCommonConnectionException(e:Exception): # This means the other side never responded. if isinstance(e, TimeoutError) and "Connection timed out" in str(e): return True - if isinstance(e, websocket.WebSocketTimeoutException): + if isinstance(e, octowebsocket.WebSocketTimeoutException): return True # This just means the server closed the socket, # or the socket connection was lost after a long delay # or there was a DNS name resolve failure. - if isinstance(e, websocket.WebSocketConnectionClosedException) and ("Connection to remote host was lost." in str(e) or "ping/pong timed out" in str(e) or "Name or service not known" in str(e)): + if isinstance(e, octowebsocket.WebSocketConnectionClosedException) and ("Connection to remote host was lost." in str(e) or "ping/pong timed out" in str(e) or "Name or service not known" in str(e)): return True # Invalid host name. - if isinstance(e, websocket.WebSocketAddressException) and "Name or service not known" in str(e): + if isinstance(e, octowebsocket.WebSocketAddressException) and "Name or service not known" in str(e): return True # We don't care. if isinstance(e. WebSocketConnectionClosedException): diff --git a/requirements.txt b/requirements.txt index 6c67803..e38785d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ # # For comments on package lock versions, see the comments in the setup.py file. # -websocket_client>=1.6.0,<1.7.99 +octowebsocket_client==1.8.2 requests>=2.31.0 octoflatbuffers==24.3.27 pillow diff --git a/setup.py b/setup.py index cf2a290..d8fb58e 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this single version string is used by all of the plugins in OctoEverywhere! -plugin_version = "3.6.0" +plugin_version = "3.6.1" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module @@ -52,12 +52,11 @@ # # On 4/13/2023 we updated to only support PY3, which frees us up from a lot of package issues. A lot the packages we depend on only support PY3 now. # -# websocket_client -# For the websocket_client, some older versions seem to have a thread issue that causes the 24 hour disconnect logic to fail, and eventually makes the thread limit get hit. -# Version 1.4.0 also has an SSL error in it. https://github.com/websocket-client/websocket-client/issues/857 -# Update: We also found a bug where the ping timer doesn't get cleaned up: https://github.com/websocket-client/websocket-client/pull/918 -# Thus we need version 1.6.0 or higher. -# The sonic pad runs python 3.7.8 as of 12/18/2023 and websocket_client>=1.7 doesn't support it. So we must keep our version at 1.6 at least for now. +# octowebsocket_client +# We forked this package so we could add a flag to disable websocket frame masking when sending messages, which got us a 30% CPU reduction. +# For a full list of changes, reasons, and version details, see the repo readme.md +# For the source lib, we must be on version 1.6 due to a bug before that version. +# We also must remain compatible with Python 3.7 for the Sonic pad. For now we are pulling the latest changes and fixing any 3.7 issues. # dnspython # We depend on a feature that was released with 2.3.0, so we need to require at least that. # For the same reason as websocket_client for the sonic pad, we also need to include at least 2.3.0, since 2.3.0 is the last version to support python 3.7.8. @@ -78,7 +77,7 @@ # # Note! These also need to stay in sync with requirements.txt, for the most part they should be the exact same! plugin_requires = [ - "websocket_client>=1.6.0,<1.7.99", + "octowebsocket_client==1.8.2", "requests>=2.31.0", "octoflatbuffers==24.3.27", "pillow", From 71d1a5f5fecf742bedde267cdce99ae3d4839345 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Tue, 24 Sep 2024 17:09:51 -0700 Subject: [PATCH 152/328] Updating the K1 script to always use the full opkg path. --- install.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/install.sh b/install.sh index f73c4ae..03ccf8a 100755 --- a/install.sh +++ b/install.sh @@ -231,8 +231,9 @@ install_or_update_system_dependencies() # We will try to update python from the package manager if possible, otherwise, we will ignore it. if [[ -f /opt/bin/opkg ]] then - opkg update || true - opkg install ${CREALITY_DEP_LIST} || true + # Use the full path to ensure it's found, since it might not be in the path if you user didn't restart the printer. + /opt/bin/opkg update || true + /opt/bin/opkg install ${CREALITY_DEP_LIST} || true fi # On the K1, the only we thing we ensure is that virtualenv is installed via pip. # We have had users report issues where this install gets stuck, using the no cache dir flag seems to fix it. From a0a6f55010194afe9101003535b593910a4de765 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Tue, 24 Sep 2024 17:14:34 -0700 Subject: [PATCH 153/328] Lint fix --- octoeverywhere/websocketimpl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octoeverywhere/websocketimpl.py b/octoeverywhere/websocketimpl.py index 05c16f2..1a82222 100644 --- a/octoeverywhere/websocketimpl.py +++ b/octoeverywhere/websocketimpl.py @@ -279,7 +279,7 @@ def IsCommonConnectionException(e:Exception): if isinstance(e, octowebsocket.WebSocketAddressException) and "Name or service not known" in str(e): return True # We don't care. - if isinstance(e. WebSocketConnectionClosedException): + if isinstance(e, octowebsocket.WebSocketConnectionClosedException): return True except Exception: pass From 270aa70dfb435125a97ace9d5805f8afa92034eb Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Mon, 30 Sep 2024 12:13:53 -0700 Subject: [PATCH 154/328] Making another perf change to add zero copy sends and zero copy reads! --- moonraker_octoeverywhere/moonrakerclient.py | 7 +++--- octoeverywhere/WebStream/octowebstream.py | 4 ++-- .../WebStream/octowebstreamhttphelper.py | 10 ++++---- .../WebStream/octowebstreamwshelper.py | 6 ++--- octoeverywhere/octoservercon.py | 4 ++-- octoeverywhere/octosessionimpl.py | 8 +++---- octoeverywhere/octostreammsgbuilder.py | 10 +++++++- octoeverywhere/websocketimpl.py | 23 +++++++++++-------- requirements.txt | 2 +- setup.py | 4 ++-- 10 files changed, 46 insertions(+), 32 deletions(-) diff --git a/moonraker_octoeverywhere/moonrakerclient.py b/moonraker_octoeverywhere/moonrakerclient.py index c936354..63588ed 100644 --- a/moonraker_octoeverywhere/moonrakerclient.py +++ b/moonraker_octoeverywhere/moonrakerclient.py @@ -346,11 +346,12 @@ def _WebSocketSend(self, jsonStr:str) -> bool: # Print for debugging. if MoonrakerClient.WebSocketMessageDebugging and self.Logger.isEnabledFor(logging.DEBUG): - self.Logger.debug("Ws ->: %s",+jsonStr) + self.Logger.debug("Ws ->: %s",jsonStr) - # Send under lock. try: - localWs.Send(jsonStr, False) + # Since we must encode the data, which will create a copy, we might as well just send the buffer as normal, + # without adding the extra space for the header. We can add the header here or in the WS lib, it's the same amount of work. + localWs.Send(jsonStr.encode("utf-8"), isData=False) except Exception as e: Sentry.Exception("Moonraker client exception in websocket send.", e) return False diff --git a/octoeverywhere/WebStream/octowebstream.py b/octoeverywhere/WebStream/octowebstream.py index 4f32c20..40da2a7 100644 --- a/octoeverywhere/WebStream/octowebstream.py +++ b/octoeverywhere/WebStream/octowebstream.py @@ -241,7 +241,7 @@ def initFromOpenMessage(self, webStreamMsg:WebStreamMsg.WebStreamMsg): # Called by the helpers to send messages to the server. - def SendToOctoStream(self, buffer, isCloseFlagSet = False, silentlyFail = False): + def SendToOctoStream(self, buffer:bytearray, msgStartOffsetBytes:int, msgSize:int, isCloseFlagSet = False, silentlyFail = False): # Make sure we aren't closed. If we are, don't allow the message to be sent. with self.StateLock: if self.IsClosed is True: @@ -263,7 +263,7 @@ def SendToOctoStream(self, buffer, isCloseFlagSet = False, silentlyFail = False) # Send now try: - self.OctoSession.Send(buffer) + self.OctoSession.Send(buffer, msgStartOffsetBytes, msgSize) except Exception as e: Sentry.Exception("Web stream "+str(self.Id)+ " failed to send a message to the OctoStream.", e) diff --git a/octoeverywhere/WebStream/octowebstreamhttphelper.py b/octoeverywhere/WebStream/octowebstreamhttphelper.py index b441d93..d342c96 100644 --- a/octoeverywhere/WebStream/octowebstreamhttphelper.py +++ b/octoeverywhere/WebStream/octowebstreamhttphelper.py @@ -49,7 +49,7 @@ def CreateBuilder(self, knownBodySizeBytes = 0): class OctoWebStreamHttpHelper: # Called by the main socket thread so this should be quick! - def __init__(self, streamId, logger:logging.Logger, webStream, webStreamOpenMsg, openedTime): + def __init__(self, streamId, logger:logging.Logger, webStream, webStreamOpenMsg:WebStreamMsg.WebStreamMsg, openedTime): self.Id = streamId self.Logger = logger self.WebStream = webStream @@ -468,12 +468,12 @@ def executeHttpRequest(self): webStreamMsgOffset = WebStreamMsg.End(builderContext.Builder) # Wrap in the OctoStreamMsg and finalize. - outputBuf = OctoStreamMsgBuilder.CreateOctoStreamMsgAndFinalize(builderContext.Builder, MessageContext.MessageContext.WebStreamMsg, webStreamMsgOffset) + buffer, msgStartOffsetBytes, msgSizeBytes = OctoStreamMsgBuilder.CreateOctoStreamMsgAndFinalize(builderContext.Builder, MessageContext.MessageContext.WebStreamMsg, webStreamMsgOffset) # Send the message. # If this is the last, we need to make sure to set that we have set the closed flag. serviceSendStartSec = time.time() - self.WebStream.SendToOctoStream(outputBuf, isLastMessage, True) + self.WebStream.SendToOctoStream(buffer, msgStartOffsetBytes, msgSizeBytes, isLastMessage, True) thisServiceSendTimeSec = time.time() - serviceSendStartSec self.ServiceUploadTimeSec += thisServiceSendTimeSec if thisServiceSendTimeSec > self.ServiceUploadTimeHighWaterMarkSec: @@ -481,9 +481,9 @@ def executeHttpRequest(self): # Do a debug check to see if our pre-allocated flatbuffer size was too small. # If this fires often, we should increase the c_MsgStreamOverheadSize size. - finalFullBufferBytes = len(builderContext.Builder.Bytes) + finalFullBufferBytes = len(buffer) if finalFullBufferBytes > lastBodyReadLength + builderContext.c_MsgStreamOverheadSize and self.Logger.isEnabledFor(logging.DEBUG): - delta = len(outputBuf) - (lastBodyReadLength + builderContext.c_MsgStreamOverheadSize) + delta = msgSizeBytes - (lastBodyReadLength + builderContext.c_MsgStreamOverheadSize) self.Logger.warn(f"The flatbuffer internal buffer had to be resized from the guess we set. Flatbuffer full buffer size: {finalFullBufferBytes}, last body read length: {lastBodyReadLength}; overrage delta: {delta}") # Clear this flag diff --git a/octoeverywhere/WebStream/octowebstreamwshelper.py b/octoeverywhere/WebStream/octowebstreamwshelper.py index bbc5c31..f1d4db9 100644 --- a/octoeverywhere/WebStream/octowebstreamwshelper.py +++ b/octoeverywhere/WebStream/octowebstreamwshelper.py @@ -300,7 +300,7 @@ def IncomingServerMessage(self, webStreamMsg:WebStreamMsg.WebStreamMsg): if self.IsWsObjClosed or self.IsClosed or localWs is None: return True # Send using the known non-null local ws object. - localWs.SendWithOptCode(buffer, sendType) + localWs.SendWithOptCode(buffer, optCode=sendType) # Log for perf tracking if self.FirstWsMessageSentToLocal is False: @@ -401,10 +401,10 @@ def onWsData(self, ws, buffer:bytes, msgType): if dataOffset is not None: WebStreamMsg.AddData(builder, dataOffset) webStreamMsgOffset = WebStreamMsg.End(builder) - outputBuf = OctoStreamMsgBuilder.CreateOctoStreamMsgAndFinalize(builder, MessageContext.MessageContext.WebStreamMsg, webStreamMsgOffset) + buffer, msgStartOffsetBytes, msgSizeBytes = OctoStreamMsgBuilder.CreateOctoStreamMsgAndFinalize(builder, MessageContext.MessageContext.WebStreamMsg, webStreamMsgOffset) # Send it! - self.WebStream.SendToOctoStream(outputBuf) + self.WebStream.SendToOctoStream(buffer, msgStartOffsetBytes, msgSizeBytes) except Exception as e: Sentry.Exception(self.getLogMsgPrefix()+ " got an error while trying to forward websocket data to the service.", e) self.WebStream.Close() diff --git a/octoeverywhere/octoservercon.py b/octoeverywhere/octoservercon.py index b4e1142..bcf33de 100644 --- a/octoeverywhere/octoservercon.py +++ b/octoeverywhere/octoservercon.py @@ -381,10 +381,10 @@ def RunBlocking(self): runForTimeChecker.Stop() - def SendMsg(self, msgBytes): + def SendMsg(self, buffer:bytearray, msgStartOffsetBytes:int, msgSize:int): # When we send any message, consider it user activity. self.LastUserActivityTime = datetime.now() - self.Ws.Send(msgBytes, True) + self.Ws.Send(buffer, msgStartOffsetBytes, msgSize, True) def GetWsId(self, ws): diff --git a/octoeverywhere/octosessionimpl.py b/octoeverywhere/octosessionimpl.py index e11a117..4544425 100644 --- a/octoeverywhere/octosessionimpl.py +++ b/octoeverywhere/octosessionimpl.py @@ -56,9 +56,9 @@ def OnSessionError(self, backoffModifierSec): self.OctoStream.OnSessionError(self.SessionId, backoffModifierSec) - def Send(self, msg): + def Send(self, buffer:bytearray, msgStartOffsetBytes:int, msgSize:int): # The message is already encoded, pass it along to the socket. - self.OctoStream.SendMsg(msg) + self.OctoStream.SendMsg(buffer, msgStartOffsetBytes, msgSize) def HandleSummonRequest(self, msg): @@ -261,12 +261,12 @@ def StartHandshake(self, summonMethod): deviceId = DeviceId.Get().GetId() # Build the message - buf = OctoStreamMsgBuilder.BuildHandshakeSyn(self.PrinterId, self.PrivateKey, self.isPrimarySession, self.PluginVersion, + buffer, msgStartOffsetBytes, msgSizeBytes = OctoStreamMsgBuilder.BuildHandshakeSyn(self.PrinterId, self.PrivateKey, self.isPrimarySession, self.PluginVersion, OctoHttpRequest.GetLocalHttpProxyPort(), LocalIpHelper.TryToGetLocalIp(), rasChallenge, rasChallengeKeyVerInt, summonMethod, self.ServerHostType, self.IsCompanion, OsTypeIdentifier.DetectOsType(), receiveCompressionType, deviceId) # Send! - self.OctoStream.SendMsg(buf) + self.OctoStream.SendMsg(buffer, msgStartOffsetBytes, msgSizeBytes) except Exception as e: Sentry.Exception("Failed to send handshake syn.", e) self.OnSessionError(0) diff --git a/octoeverywhere/octostreammsgbuilder.py b/octoeverywhere/octostreammsgbuilder.py index 572f452..12ecf28 100644 --- a/octoeverywhere/octostreammsgbuilder.py +++ b/octoeverywhere/octostreammsgbuilder.py @@ -64,7 +64,15 @@ def CreateOctoStreamMsgAndFinalize(builder, contextType, contextOffset): # Finalize the message. We use the size prefixed builder.FinishSizePrefixed(streamMsgOffset) - return builder.Output() + + # Instead of using Output, which will create a copy of the buffer that's trimmed, we return the fully built buffer + # with the header offset set and size. Flatbuffers are built backwards, so there's usually space in the front were we can add data + # without creating a new buffer! + # Note that the buffer is a bytearray + buffer = builder.Bytes + msgStartOffsetBytes = builder.Head() + return (buffer, msgStartOffsetBytes, len(buffer) - msgStartOffsetBytes) + #return builder.Output() @staticmethod def BytesToString(buf) -> str: diff --git a/octoeverywhere/websocketimpl.py b/octoeverywhere/websocketimpl.py index 1a82222..465d495 100644 --- a/octoeverywhere/websocketimpl.py +++ b/octoeverywhere/websocketimpl.py @@ -150,7 +150,7 @@ def _Close(self): # Always ensure we close the send queue. try: # Push an empty buffer to the send queue, which will close it. - self.SendQueue.put(SendQueueContext(None, None)) + self.SendQueue.put(SendQueueContext(None)) except Exception as e: Sentry.Exception("Exception while trying to close the send queue.", e) @@ -189,19 +189,22 @@ def fireWsErrorCallbackThread(self, exception): self._Close() - def Send(self, msgBytes, isData): + def Send(self, buffer:bytearray, msgStartOffsetBytes:int = None, msgSize:int = None, isData:bool = True): if isData: - self.SendWithOptCode(msgBytes, octowebsocket.ABNF.OPCODE_BINARY) + self.SendWithOptCode(buffer, msgStartOffsetBytes, msgSize, octowebsocket.ABNF.OPCODE_BINARY) else: - self.SendWithOptCode(msgBytes, octowebsocket.ABNF.OPCODE_TEXT) + self.SendWithOptCode(buffer, msgStartOffsetBytes, msgSize, octowebsocket.ABNF.OPCODE_TEXT) - def SendWithOptCode(self, msgBytes, opcode): + # Sends a buffer, with an optional message start offset and size. + # If the message start offset and size are not provided, it's assumed the buffer starts at 0 and the size is the full buffer. + # Providing a bytearray with room in the front allows the system to avoid copying the buffer. + def SendWithOptCode(self, buffer:bytearray, msgStartOffsetBytes:int = None, msgSize:int = None, optCode = octowebsocket.ABNF.OPCODE_BINARY): try: # Make sure we have a buffer, this is invalid and it will also shutdown our send thread. - if msgBytes is None: + if buffer is None: raise Exception("We tired to send a message to the websocket with a None buffer.") - self.SendQueue.put(SendQueueContext(msgBytes, opcode)) + self.SendQueue.put(SendQueueContext(buffer, msgStartOffsetBytes, msgSize, optCode)) except Exception as e: # If any exception happens during sending, we want to report the error # and shutdown the entire websocket. @@ -220,7 +223,7 @@ def _SendQueueThread(self): # Important! We don't want to use the frame mask because it adds about 30% CPU usage on low end devices. # The frame masking was only need back when websockets were used over the internet without SSL. # Our server, OctoPrint, and Moonraker all accept unmasked frames, so its safe to do this for all WS. - self.Ws.send(context.Buffer, context.OptCode, False) + self.Ws.send(context.Buffer, context.OptCode, False, context.MsgStartOffsetBytes, context.MsgSize) except Exception as e: # If any exception happens during sending, we want to report the error # and shutdown the entire websocket. @@ -287,6 +290,8 @@ def IsCommonConnectionException(e:Exception): class SendQueueContext(): - def __init__(self, buffer, optCode) -> None: + def __init__(self, buffer:bytearray, msgStartOffsetBytes:int = None, msgSize:int = None, optCode = octowebsocket.ABNF.OPCODE_BINARY) -> None: self.Buffer = buffer + self.MsgStartOffsetBytes = msgStartOffsetBytes + self.MsgSize = msgSize self.OptCode = optCode diff --git a/requirements.txt b/requirements.txt index e38785d..49f2c2d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ # # For comments on package lock versions, see the comments in the setup.py file. # -octowebsocket_client==1.8.2 +octowebsocket_client==1.8.3 requests>=2.31.0 octoflatbuffers==24.3.27 pillow diff --git a/setup.py b/setup.py index d8fb58e..bfe6415 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this single version string is used by all of the plugins in OctoEverywhere! -plugin_version = "3.6.1" +plugin_version = "3.6.2" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module @@ -77,7 +77,7 @@ # # Note! These also need to stay in sync with requirements.txt, for the most part they should be the exact same! plugin_requires = [ - "octowebsocket_client==1.8.2", + "octowebsocket_client==1.8.3", "requests>=2.31.0", "octoflatbuffers==24.3.27", "pillow", From 591d370450f5da5b8bf3b15ef2da74e576f97f4b Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sat, 5 Oct 2024 10:27:40 -0700 Subject: [PATCH 155/328] Reverting out the Bambu Cloud install from the docker readme. --- docker-compose.yml | 30 ++++--------------- docker-readme.md | 73 +++++++++++++--------------------------------- 2 files changed, 26 insertions(+), 77 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 560a77e..b2a8f3b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,46 +1,28 @@ +version: '2' services: octoeverywhere-bambu-connect: image: octoeverywhere/octoeverywhere:latest environment: + # https://octoeverywhere.com/s/access-code + - ACCESS_CODE=XXXXXXXX # https://octoeverywhere.com/s/bambu-sn - SERIAL_NUMBER=XXXXXXXXXXXXXXX # Find using the printer's display or use https://octoeverywhere.com/s/bambu-ip - PRINTER_IP=XXX.XXX.XXX.XXX - - # ~~~ If connecting with Bambu Cloud Mode ~~~ - # https://octoeverywhere.com/s/bambu-setup - - BAMBU_CLOUD_ACCOUNT_EMAIL=XXXXXXXX - - BAMBU_CLOUD_ACCOUNT_PASSWORD=XXXXXXXX - #- BAMBU_CLOUD_REGION=china # Optional, use if your Bambu account is in the China region - - # ~~~ OR If connecting with LAN Only Mode ~~~ - # https://octoeverywhere.com/s/access-code - # - ACCESS_CODE=XXXXXXXX - # - LAN_ONLY_MODE=TRUE volumes: - # Specify a path mapping for the required persistent storage # This can also be an absolue path, e.g. /var/octoeverywhere/plugin/data or /c/users/name/plugin/data - ./data:/data - # Add as many printers as you want! Just make the name `octoeverywhere-bambu-connect` unique! + # Add as many printers as you want! # octoeverywhere-bambu-connect-2: # image: octoeverywhere/octoeverywhere:latest # environment: + # # https://octoeverywhere.com/s/access-code + # - ACCESS_CODE=XXXXXXXX # # https://octoeverywhere.com/s/bambu-sn # - SERIAL_NUMBER=XXXXXXXXXXXXXXX # # Find using the printer's display or use https://octoeverywhere.com/s/bambu-ip # - PRINTER_IP=XXX.XXX.XXX.XXX - - # # ~~~ If connecting with Bambu Cloud Mode ~~~ - # # https://octoeverywhere.com/s/bambu-setup - # - BAMBU_CLOUD_ACCOUNT_EMAIL=XXXXXXXX - # - BAMBU_CLOUD_ACCOUNT_PASSWORD=XXXXXXXX - # #- BAMBU_CLOUD_REGION=china # Optional, use if your Bambu account is in the China region - - # # ~~~ OR If connecting with LAN Only Mode ~~~ - # # https://octoeverywhere.com/s/access-code - # # - ACCESS_CODE=XXXXXXXX - # # - LAN_ONLY_MODE=TRUE # volumes: # # Specify a path mapping for the required persistent storage # - ./data:/data \ No newline at end of file diff --git a/docker-readme.md b/docker-readme.md index e44e639..a43a7fb 100644 --- a/docker-readme.md +++ b/docker-readme.md @@ -1,53 +1,30 @@ # Bambu Connect Docker Support -OctoEverywhere's docker image only works with [Bambu Connect](https://octoeverywhere.com/bambu?source=github_docker_readme) for Bambu Lab 3D printers. If you are using OctoPrint or Klipper, [follow our getting started guide to install the OctoEverywhere plugin.](https://octoeverywhere.com/getstarted?source=github_docker_readme) +OctoEverywhere's docker image only works for [Bambu Connect](https://octoeverywhere.com/bambu?source=github_docker_readme) for Bambu Lab 3D printers. If you are using OctoPrint or Klipper, [follow our getting started guide](https://octoeverywhere.com/getstarted?source=github_docker_readme) to install the OctoEverywhere plugin. Official Docker Image: https://hub.docker.com/r/octoeverywhere/octoeverywhere +## Required Setup Environment Vars -## Bambu Cloud Vs Lan Only Connection Modes - -Bambu Lab made a firmware change in July 2024 where 3rd-party addons can't connect to the printer directly over your LAN network if the printer is connected to Bambu Cloud. - -Thus, you can pick either of these install methods: - -1) Connect OctoEverywhere to your 3D printer through Bambu Cloud. -2) Put your 3D printer in "LAN Only Mode" and connect OctoEverywhere locally to the 3D printer. - -Note if you put your printer in "LAN Only Mode" you **can** still use Bambu Studio and Bambu Handy when on the same network as the 3D printer. - -### Connect Via Bambu Cloud - -For OctoEverywhere to connect to your 3D printer through Bambu Cloud, you just need to supply your Bambu Cloud account info to the local plugin. - -**Rest assured, your Bambu Cloud email address and password are stored locally, secured on disk, and are never sent to the OctoEverywhere service.** - -If you use Facebook, Google, or Apple to login to Bambu Cloud, [follow this guide to set a password on your account.](https://intercom.help/octoeverywhere/en/articles/9529936-bambu-cloud-with-bambu-connect) - - -### Connect Via 'LAN Only Mode' +To use the Bambu Connect plugin, you need to get the following information. -If you don't mind disabling the Bambu Cloud, you can enable "LAN only mode" on your Bambu Lab 3D printer. +- Your printer's Access Code - https://octoeverywhere.com/s/access-code +- Your printer's Serial Number - https://octoeverywhere.com/s/bambu-sn +- Your printer's IP Address - (use the printer's display) -In "LAN only mode" OctoEverywhere can directly connect to your 3D printer on you local network, there's no need to supply your Bambu Cloud email or password. With Bambu Cloud disabled, you WILL still be able to use Bambu Studio and Bambu Handy while on the same network as your 3D printer. +These three values must be set at environment vars when you first run the container. Once the container is run, you don't need to include them again, unless you want to update the values. -## Required Setup Information +- ACCESS_CODE=(code) +- SERIAL_NUMBER=(serial number) +- PRINTER_IP=(ip address) -To use the Bambu Connect plugin, you need to get the following information. +## Required Persistent Storage -- Your printer's Serial Number - https://octoeverywhere.com/s/bambu-sn -- Your printer's IP Address - Use the printer's display or https://octoeverywhere.com/s/bambu-ip -- If you're connecting with Bambu Cloud... - - Your Bambu Cloud account email address - - Your Bambu Cloud account password - - **Note:** Your Bambu Cloud email address and password are stored locally, secured on disk, and never sent to the OctoEverywhere service. - - Learn more here: https://octoeverywhere.com/s/bambu-setup -- Or if you're connecting in LAN Only Mode... - - Your printer's Access Code - https://octoeverywhere.com/s/access-code +You must map the `/data` folder in your docker container to a directory on your computer so the plugin can write data that will remain between runs. Failure to do this will require relinking the plugin when the container is destroyed or updated. ## Linking Your Bambu Connect Plugin -Once the docker container is running, you need to view the logs to find the linking URL. +Once the docker container is running, you need to look at the logs to find the linking URL. Docker Compose: `docker compose logs | grep https://octoeverywhere.com/getstarted` @@ -63,33 +40,23 @@ Using docker compose is the easiest way to run OctoEverywhere's Bambu Connect us - Install [Docker and Docker Compose](https://docs.docker.com/compose/install/linux/) - Clone this repo -- Edit the `./docker-compose.yml` file to enter your environment information.. +- Edit the `./docker-compose.yml` file to enter your environment vars - Run `docker compose up -d` - Follow the "Linking Your Bambu Connect Plugin" to link the plugin to your account. ## Using Docker -These three values must be set at environment vars when you first run the container. Once the container is ran, you don't need to include them, unless you want to update the values. - -- SERIAL_NUMBER=(serial number) -- PRINTER_IP=(ip address) -- If connecting via Bambu Cloud... - - BAMBU_CLOUD_ACCOUNT_EMAIL=(email) - - BAMBU_CLOUD_ACCOUNT_PASSWORD=(password) - - Optional - BAMBU_CLOUD_REGION=china - Use if your Bambu account is in the China region. -- If connecting via LAN Only Mode... - - ACCESS_CODE=(code) - - LAN_ONLY_MODE=TRUE +Docker compose is a fancy wrapper to run docker containers. You can also run docker containers manually. -Run the docker container passing the required information: +Use a command like this example, but update the required vars. -`docker run --name bambu-connect -e SERIAL_NUMBER= -e PRINTER_IP= -e BAMBU_CLOUD_ACCOUNT_EMAIL="" -e BAMBU_CLOUD_ACCOUNT_PASSWORD="" -v ./data:/data -d octoeverywhere/octoeverywhere` -`docker run --name bambu-connect -e SERIAL_NUMBER=test -e PRINTER_IP=1.1.1.1 -e LAN_ONLY_MODE=1 -v /data:/data -d octoeverywhere/octoeverywhere` +`docker pull octoeverywhere/octoeverywhere` +`docker run --name bambu-connect -e ACCESS_CODE= -e SERIAL_NUMBER= -e PRINTER_IP= -v /your/local/path:/data -d octoeverywhere/octoeverywhere` Follow the "Linking Your Bambu Connect Plugin" to link the plugin to your account. ## Building The Image Locally -You can build the docker image locally if you prefer; use the following command. +You can build the docker image locally if you prefer, use the following command. -`docker build -t octoeverywhere-local .` \ No newline at end of file +`docker build -t octoeverywhere .` \ No newline at end of file From e02d38f3718413b4683fa9e0ce75c063c17c706b Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sat, 5 Oct 2024 10:28:29 -0700 Subject: [PATCH 156/328] One more update to the docker readme. --- docker-readme.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docker-readme.md b/docker-readme.md index a43a7fb..fbb35eb 100644 --- a/docker-readme.md +++ b/docker-readme.md @@ -1,6 +1,6 @@ # Bambu Connect Docker Support -OctoEverywhere's docker image only works for [Bambu Connect](https://octoeverywhere.com/bambu?source=github_docker_readme) for Bambu Lab 3D printers. If you are using OctoPrint or Klipper, [follow our getting started guide](https://octoeverywhere.com/getstarted?source=github_docker_readme) to install the OctoEverywhere plugin. +OctoEverywhere's docker image only works with [Bambu Connect](https://octoeverywhere.com/bambu?source=github_docker_readme) for Bambu Lab 3D printers. If you are using OctoPrint or Klipper, [follow our getting started guide to install the OctoEverywhere plugin.](https://octoeverywhere.com/getstarted?source=github_docker_readme) Official Docker Image: https://hub.docker.com/r/octoeverywhere/octoeverywhere @@ -40,7 +40,7 @@ Using docker compose is the easiest way to run OctoEverywhere's Bambu Connect us - Install [Docker and Docker Compose](https://docs.docker.com/compose/install/linux/) - Clone this repo -- Edit the `./docker-compose.yml` file to enter your environment vars +- Edit the `./docker-compose.yml` file to enter your environment information. - Run `docker compose up -d` - Follow the "Linking Your Bambu Connect Plugin" to link the plugin to your account. @@ -50,7 +50,6 @@ Docker compose is a fancy wrapper to run docker containers. You can also run doc Use a command like this example, but update the required vars. -`docker pull octoeverywhere/octoeverywhere` `docker run --name bambu-connect -e ACCESS_CODE= -e SERIAL_NUMBER= -e PRINTER_IP= -v /your/local/path:/data -d octoeverywhere/octoeverywhere` Follow the "Linking Your Bambu Connect Plugin" to link the plugin to your account. From 2533f687cd7eb3c9893715654d383c574daa97df Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sun, 3 Nov 2024 20:31:14 -0800 Subject: [PATCH 157/328] Fixing a minor bug in the plugin were redirects would cause a client disconnect. --- octoeverywhere/WebStream/octowebstreamhttphelper.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/octoeverywhere/WebStream/octowebstreamhttphelper.py b/octoeverywhere/WebStream/octowebstreamhttphelper.py index d342c96..c80432e 100644 --- a/octoeverywhere/WebStream/octowebstreamhttphelper.py +++ b/octoeverywhere/WebStream/octowebstreamhttphelper.py @@ -359,7 +359,7 @@ def executeHttpRequest(self): nonCompressedBodyReadSize = 0 lastBodyReadLength = 0 dataOffset = None - compressBody = False + # Note that compressBody will be set to false in the special case below. else: # Start by reading data from the response. # This function will return a read length of 0 and a null data offset if there's nothing to read. @@ -399,6 +399,14 @@ def executeHttpRequest(self): # - We have an expected length and we have hit it or gone over it. isLastMessage = dataOffset is None or (contentLength is not None and nonCompressedContentReadSizeBytes >= contentLength) + # Special Case - If this request has no body, we need to make sure we the `compressBody` flag is set to false. + # For example, if this request is not 200 but has no content, compressBody might be set but we didn't read any body, so we didn't compress anything, + # and thus self.CompressionType will not be set. + if isLastMessage and nonCompressedContentReadSizeBytes == 0: + # TODO - Remove this log after we are sure this is working correctly. + self.Logger.warn(self.getLogMsgPrefix()+" read no body so we will turned off the compressBody flag.") + compressBody = False + # If this is the first response in the stream, we need to send the initial http context and status code. httpInitialContextOffset = None statusCode = None From 2c1e39786358903d857e7b6cb7949b3aa0a2c5d1 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sun, 3 Nov 2024 20:55:57 -0800 Subject: [PATCH 158/328] Version bump! --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bfe6415..57b6356 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this single version string is used by all of the plugins in OctoEverywhere! -plugin_version = "3.6.2" +plugin_version = "3.6.3" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 7e9b870744595f21379c6203835a531a579de95b Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Thu, 7 Nov 2024 06:13:14 -0800 Subject: [PATCH 159/328] Minor logging tweaks --- octoeverywhere/WebStream/octowebstreamhttphelper.py | 3 +-- octoeverywhere/octoservercon.py | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/octoeverywhere/WebStream/octowebstreamhttphelper.py b/octoeverywhere/WebStream/octowebstreamhttphelper.py index c80432e..3067bcf 100644 --- a/octoeverywhere/WebStream/octowebstreamhttphelper.py +++ b/octoeverywhere/WebStream/octowebstreamhttphelper.py @@ -403,8 +403,7 @@ def executeHttpRequest(self): # For example, if this request is not 200 but has no content, compressBody might be set but we didn't read any body, so we didn't compress anything, # and thus self.CompressionType will not be set. if isLastMessage and nonCompressedContentReadSizeBytes == 0: - # TODO - Remove this log after we are sure this is working correctly. - self.Logger.warn(self.getLogMsgPrefix()+" read no body so we will turned off the compressBody flag.") + self.Logger.debug(self.getLogMsgPrefix()+" read no body so we will turned off the compressBody flag.") compressBody = False # If this is the first response in the stream, we need to send the initial http context and status code. diff --git a/octoeverywhere/octoservercon.py b/octoeverywhere/octoservercon.py index bcf33de..c5ca544 100644 --- a/octoeverywhere/octoservercon.py +++ b/octoeverywhere/octoservercon.py @@ -183,6 +183,7 @@ def OnHandshakeComplete(self, sessionId, octoKey, connectedAccounts): self.Logger.info("Handshake complete, server con "+self.GetConnectionString()+", successfully connected to OctoEverywhere!") # Only primary connections have this handler. + # For secondary connections, octoKey and connectedAccounts will be None. if self.StatusChangeHandler is not None: self.StatusChangeHandler.OnPrimaryConnectionEstablished(octoKey, connectedAccounts) From d17a24130bde405d29925ae57131bf138e931d39 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sat, 9 Nov 2024 14:24:50 -0800 Subject: [PATCH 160/328] Fixing a status input bounds bug. --- bambu_octoeverywhere/bambucommandhandler.py | 3 ++- moonraker_octoeverywhere/moonrakercommandhandler.py | 3 ++- octoprint_octoeverywhere/octoprintcommandhandler.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/bambu_octoeverywhere/bambucommandhandler.py b/bambu_octoeverywhere/bambucommandhandler.py index 96c2a8a..9242bbc 100644 --- a/bambu_octoeverywhere/bambucommandhandler.py +++ b/bambu_octoeverywhere/bambucommandhandler.py @@ -185,7 +185,8 @@ def GetCurrentJobStatus(self): { "Progress" : progress, "DurationSec" : durationSec, - "TimeLeftSec" : timeLeftSec, + # In some system buggy cases, the time left can be super high and won't fit into a int32, so we cap it. + "TimeLeftSec" : min(timeLeftSec, 2147483600), "FileName" : fileName, "EstTotalFilUsedMm" : filamentUsageMm, "CurrentLayer": currentLayerInt, diff --git a/moonraker_octoeverywhere/moonrakercommandhandler.py b/moonraker_octoeverywhere/moonrakercommandhandler.py index f6e153e..a478839 100644 --- a/moonraker_octoeverywhere/moonrakercommandhandler.py +++ b/moonraker_octoeverywhere/moonrakercommandhandler.py @@ -142,7 +142,8 @@ def GetCurrentJobStatus(self): { "Progress" : progress, "DurationSec" : durationSec, - "TimeLeftSec" : timeLeftSec, + # In some system buggy cases, the time left can be super high and won't fit into a int32, so we cap it. + "TimeLeftSec" : min(timeLeftSec, 2147483600), "FileName" : fileName, "EstTotalFilUsedMm" : filamentUsageMm, "CurrentLayer": currentLayerInt, diff --git a/octoprint_octoeverywhere/octoprintcommandhandler.py b/octoprint_octoeverywhere/octoprintcommandhandler.py index 81aec6a..d26ba92 100644 --- a/octoprint_octoeverywhere/octoprintcommandhandler.py +++ b/octoprint_octoeverywhere/octoprintcommandhandler.py @@ -125,7 +125,8 @@ def GetCurrentJobStatus(self): { "Progress" : progress, "DurationSec" : durationSec, - "TimeLeftSec" : timeLeftSec, + # In some system buggy cases, the time left can be super high and won't fit into a int32, so we cap it. + "TimeLeftSec" : min(timeLeftSec, 2147483600), "FileName" : fileName, "EstTotalFilUsedMm" : estTotalFilamentUsageMm, "CurrentLayer": None, # OctoPrint doesn't provide these. From c4660b3930c22f4e90f49f15692bfbc8b4cf4acd Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sat, 9 Nov 2024 14:28:38 -0800 Subject: [PATCH 161/328] Updating the debug thread logger with more info. --- octoeverywhere/threaddebug.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/octoeverywhere/threaddebug.py b/octoeverywhere/threaddebug.py index f102cb6..ea3186f 100644 --- a/octoeverywhere/threaddebug.py +++ b/octoeverywhere/threaddebug.py @@ -32,14 +32,14 @@ def DoThreadDumpLogout(logger:logging.Logger): # pylint: disable=protected-access for threadId, stack in sys._current_frames().items(): trace = "" - for filename, _, name, _ in traceback.extract_stack(stack): + for filename, lineno, name, line in traceback.extract_stack(stack): parts = filename.split("\\") if len(parts) == 0: parts = filename.split("/") if len(parts) > 0: - trace += ", "+parts[len(parts)-1]+":"+name + trace += f", {parts[len(parts)-1]}:{lineno}={name}:{line}" else: - trace += ", "+filename+":"+name + trace += f", {filename}:{lineno}={name}:{line}" logger.info("ThreadDump- Id: "+str(threadId) + " -> "+str(trace)) except Exception as e: logger.error("Exception in ThreadDebug : "+str(e)) From b5778b1651b0332a2c94a07d721918c06e19432d Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Mon, 25 Nov 2024 13:12:48 -0800 Subject: [PATCH 162/328] Adding better logic for unknown http body size reads. --- octoeverywhere/WebStream/octowebstream.py | 4 +- .../WebStream/octowebstreamhttphelper.py | 257 +++++++++++++----- octoeverywhere/octohttprequest.py | 2 + setup.py | 2 +- 4 files changed, 191 insertions(+), 74 deletions(-) diff --git a/octoeverywhere/WebStream/octowebstream.py b/octoeverywhere/WebStream/octowebstream.py index 40da2a7..873f890 100644 --- a/octoeverywhere/WebStream/octowebstream.py +++ b/octoeverywhere/WebStream/octowebstream.py @@ -291,9 +291,9 @@ def ensureCloseMessageSent(self): WebStreamMsg.AddIsCloseMsg(builder, True) WebStreamMsg.AddCloseDueToRequestConnectionFailure(builder, self.ClosedDueToRequestConnectionError) webStreamMsgOffset = WebStreamMsg.End(builder) - outputBuf = OctoStreamMsgBuilder.CreateOctoStreamMsgAndFinalize(builder, MessageContext.MessageContext.WebStreamMsg, webStreamMsgOffset) + buffer, msgStartOffsetBytes, msgSizeBytes = OctoStreamMsgBuilder.CreateOctoStreamMsgAndFinalize(builder, MessageContext.MessageContext.WebStreamMsg, webStreamMsgOffset) # Set the flag to silently fail, since the message might have already been sent by the helper. - self.SendToOctoStream(outputBuf, True, True) + self.SendToOctoStream(buffer, msgStartOffsetBytes, msgSizeBytes, True, True) except Exception as e: # This is bad, log it and kill the stream. Sentry.Exception("Exception thrown while trying to send close message for web stream "+str(self.Id), e) diff --git a/octoeverywhere/WebStream/octowebstreamhttphelper.py b/octoeverywhere/WebStream/octowebstreamhttphelper.py index 3067bcf..5ff521c 100644 --- a/octoeverywhere/WebStream/octowebstreamhttphelper.py +++ b/octoeverywhere/WebStream/octowebstreamhttphelper.py @@ -1,7 +1,6 @@ -# namespace: WebStream - import time import logging +import threading import requests import urllib3 @@ -72,9 +71,9 @@ def __init__(self, streamId, logger:logging.Logger, webStream, webStreamOpenMsg: self.UploadBytesReceivedSoFar = 0 self.UploadBuffer = None - # Micro body read stuff. - self.IsDoingMicroBodyReads = False - self.IsFirstMicroBodyRead = True + # Unknown body size chunk reader + # If this is not None, we are doing the unknown body read. Then the rest of the body reads must use this same system. + self.UnknownBodyChunkReadContext:UnknownBodyChunkReadContext = None # Perf stats self.BodyReadTimeSec = 0.0 @@ -100,8 +99,17 @@ def __init__(self, streamId, logger:logging.Logger, webStream, webStreamOpenMsg: # When close is called, all http operations should be shutdown. # Called by the main socket thread so this should be quick! def Close(self): + # Set the flag so all of the looping http operations will stop. self.IsClosed = True + # Important! If we are doing a unknown chunk read, we need to set the wait event to unblock the stream read thread. + # This will cause the thread to wake up, it will see the IsClosed flag, return, and then allow the web request to close, + # which will end the unknown body read thread. + if self.UnknownBodyChunkReadContext is not None: + # Call set under lock, to ensure the other thread doesn't clear it without us seeing it. + with self.UnknownBodyChunkReadContext.BufferLock: + self.UnknownBodyChunkReadContext.BufferDataReadyEvent.set() + # Called when a new message has arrived for this stream from the server. # This function should throw on critical errors, that will reset the connection. @@ -272,12 +280,12 @@ def executeHttpRequest(self): # Look at the headers to see what kind of response we are dealing with. # See if we find a content length, for http request that are streams, there is no content length. - contentLength = None + contentLength:int = None # We will also look for the content type, and look for a boundary string if there is one # The boundary stream is used for webcam streams, and it's an ideal place to package and send each frame - boundaryStr = None + boundaryStr:str = None # Pull out the content type value, so we can use it to figure out if we want to compress this data or not - contentTypeLower =None + contentTypeLower:str =None headers = octoHttpResult.Headers for name, value in headers.items(): nameLower = name.lower() @@ -500,7 +508,7 @@ def executeHttpRequest(self): # Log about it - only if debug is enabled. Otherwise, we don't want to waste time making the log string. responseWriteDone = time.time() if self.Logger.isEnabledFor(logging.DEBUG): - self.Logger.debug(self.getLogMsgPrefix() + method+" [upload:"+str(format(requestExecutionStart - self.OpenedTime, '.3f'))+"s; request_exe:"+str(format(requestExecutionEnd - requestExecutionStart, '.3f'))+"s; send:"+str(format(responseWriteDone - requestExecutionEnd, '.3f'))+"s; body_read:"+str(format(self.BodyReadTimeSec, '.3f'))+"s; compress:"+str(format(self.CompressionTimeSec, '.3f'))+"s; octo_stream_upload:"+str(format(self.ServiceUploadTimeSec, '.3f'))+"s] size:("+str(nonCompressedContentReadSizeBytes)+"->"+str(contentReadBytes)+") compressed:"+str(compressBody)+" msgcount:"+str(messageCount)+" microreads:"+str(self.IsDoingMicroBodyReads)+" type:"+str(contentTypeLower)+" status:"+str(octoHttpResult.StatusCode)+" cached:"+str(isFromCache)+" for " + uri) + self.Logger.debug(self.getLogMsgPrefix() + method+" [upload:"+str(format(requestExecutionStart - self.OpenedTime, '.3f'))+"s; request_exe:"+str(format(requestExecutionEnd - requestExecutionStart, '.3f'))+"s; send:"+str(format(responseWriteDone - requestExecutionEnd, '.3f'))+"s; body_read:"+str(format(self.BodyReadTimeSec, '.3f'))+"s; compress:"+str(format(self.CompressionTimeSec, '.3f'))+"s; octo_stream_upload:"+str(format(self.ServiceUploadTimeSec, '.3f'))+"s] size:("+str(nonCompressedContentReadSizeBytes)+"->"+str(contentReadBytes)+") compressed:"+str(compressBody)+" msgcount:"+str(messageCount)+" microreads:"+str(self.UnknownBodyChunkReadContext is not None)+" type:"+str(contentTypeLower)+" status:"+str(octoHttpResult.StatusCode)+" cached:"+str(isFromCache)+" for " + uri) def buildHeaderVector(self, builder, octoHttpResult:OctoHttpRequest.Result): @@ -747,7 +755,7 @@ def shouldCompressBody(self, contentTypeLower:str, octoHttpResult:OctoHttpReques # Reads data from the response body, puts it in a data vector, and returns the offset. # If the body has been fully read, this should return ogLen == 0, len = 0, and offset == None # The read style depends on the presence of the boundary string existing. - def readContentFromBodyAndMakeDataVector(self, builderContext:MsgBuilderContext, octoHttpResult:OctoHttpRequest.Result, boundaryStr_opt, shouldCompress, contentTypeLower_NoneIfNotKnown, contentLength_NoneIfNotKnown, responseHandlerContext): + def readContentFromBodyAndMakeDataVector(self, builderContext:MsgBuilderContext, octoHttpResult:OctoHttpRequest.Result, boundaryStr_opt, shouldCompress, contentTypeLower_NoneIfNotKnown:str, contentLength_NoneIfNotKnown:int, responseHandlerContext): # This is the max size each body read will be. Since we are making local calls, most of the time we will always get this full amount as long as theres more body to read. # This size is a little under the max read buffer on the server, allowing the server to handle the buffers with no copies. # @@ -798,13 +806,10 @@ def readContentFromBodyAndMakeDataVector(self, builderContext:MsgBuilderContext, finalDataBufferMv_CanBeNone = memoryview(self.BodyReadTempBuffer) finalDataBuffer = finalDataBufferMv_CanBeNone[0:readLength] else: - if responseHandlerContext is None and self.shouldDoUnknownBodySizeRead(contentTypeLower_NoneIfNotKnown, contentLength_NoneIfNotKnown): - # If we don't know the content length AND there is no boundary string, this request is probably a event stream of some sort. - # We have to use this special read function, because doBodyRead will block until the full buffer is filled, which might take a long time - # for a number of streamed messages to fill it up. This special function does micro reads on the socket until a time limit is hit, and then - # returns what was received. - self.IsDoingMicroBodyReads = True - finalDataBuffer = self.doUnknownBodySizeRead(octoHttpResult) + if self.UnknownBodyChunkReadContext is not None or (responseHandlerContext is None and self.shouldDoUnknownBodyChunkRead(contentTypeLower_NoneIfNotKnown, contentLength_NoneIfNotKnown)): + # According to the HTTP 1.1 spec, if there's no content length and no boundary string, then the body is chunk based transfer encoding. + # Note that once we do on read as an unknown body size chunk read, we need to always do it, since there's a thread reading the body. + finalDataBuffer = self.doUnknownBodyChunkRead(octoHttpResult) else: # If there is no boundary string, but we know the content length, it's safe to just read. # This will block until either the full defaultBodyReadSizeBytes is read or the full request has been received. @@ -1099,73 +1104,161 @@ def doBodyRead(self, octoHttpResult:OctoHttpRequest.Result, readSize:int): Sentry.Exception(self.getLogMsgPrefix()+ " exception thrown in doBodyRead. Ending body read.", e) return None - # This is similar to doBodyRead, but it allows us to send chunks of the body over time. - # The problem is for requests where the content length isn't known AND there's no boundary string, response.raw.read(size) will block - # until the full amount of data requested is read. That doesn't work for things like event streams, because there's no boundary string and the full length is unknown, - # but we want to stream the data as it arrives to us. To make doBodyRead efficient, we request a large read buffer, so if the event stream contains many small messages, - # doBodyRead will block until it accumulates enough messages to fill the full buffer and send it. - # - # For normal requests with known content lengths, response.raw.read will read full buffers until the full content is known to be done, and then will return the final subset buffer, - # so they can't get blocked like streaming event can. So if the event stream isn't sending data often, this can get stuck while waiting for the final bytes of a message. + + def doUnknownBodyChunkReadThread(self): + try: + if self.Logger.isEnabledFor(logging.DEBUG): + self.Logger.debug(f"{self.getLogMsgPrefix()}Starting chunk read thread.") + + # Get the Request response object. + response = self.UnknownBodyChunkReadContext.HttpResult.ResponseForBodyRead + if response is None: + raise Exception("doUnknownBodyChunkReadThread was called with a result that has not Response object to read from.") + + # Loop until the stream is closed. + # Remember we use the raw stream read, because it will read and entire chunk and return it as soon as it's ready. + # BUG - response.raw.stream doesn't close when we close the http request from our side. (but it says it should?) + # If server we are calling closes it shutdown correctly, but if our server drops the connection it will not close. + # So what happens is that the stream will timeout from the httprequest.MakeHttpCallAttempt timeout, or when it gets a new chunk it will end. + # That's not great, but it's not super common, so it's fine. + gen = response.raw.stream(amt=None) + for i in gen: + # When we have a new buffer, add it to the list under lock. + with self.UnknownBodyChunkReadContext.BufferLock: + self.UnknownBodyChunkReadContext.BufferList.append(i) + # Call set under lock, to ensure the other thread doesn't clear it without us seeing it. + self.UnknownBodyChunkReadContext.BufferDataReadyEvent.set() + + # When the loop exits, the body read is complete and the stream is closed. + + except Exception as e: + # If the web stream is already closed, don't bother logging the exception. + # These exceptions happen for use cases as above, where stream() doesn't close in time and such. + # Note the exception can be a timeout, but it can also be a "doesn't have a read" function error bc if the socket gets data the lib will try to call read on a fp that's closed and set to None. :/ + if self.IsClosed is False: + Sentry.Exception(self.getLogMsgPrefix()+ " exception thrown in doUnknownBodyChunkReadThread", e) + finally: + # Ensure we always set this flag, so the web stream will know the body read is done. + self.UnknownBodyChunkReadContext.ReadComplete = True + + # Set the event to break the stream read wait, so it will shutdown. + # Call set under lock, to ensure the other thread doesn't clear it without us seeing it. + with self.UnknownBodyChunkReadContext.BufferLock: + self.UnknownBodyChunkReadContext.BufferDataReadyEvent.set() + + try: + if self.Logger.isEnabledFor(logging.DEBUG): + self.Logger.debug(f"{self.getLogMsgPrefix()}Exiting chunk read thread.") + except Exception: + pass + + + # This function should be used if there's no content length and there's no boundary string. + # In that case, the HTTP1.1 standard says the body content must be chunk based transfer encoded, which is what this function does. + # Most of the time these HTTP calls are for streams, like an event stream, log stream, etc. # - # Event streams are an important fallback for OctoPrint, and is also what OctoFarm uses to stream instead of websockets. + # In the past we had two problems with this: + # 1) A body read() will block until the size requested is full, which means we can't stream chunks as they come in. + # 2) We then tired to do micro reads to build a buffer but it allowed us to return if there was enough. But this still failed because read still needs to fill the buffer, + # so if we wanted to read the last 5 bytes of the buffer but set our size to 10, it would block until the final 5 bytes were read. # - # Thus, this function does many small reads (which isn't as efficient) but builds them into a larger buffer that's time limited. In this way, we ensure we are still streaming - # messages every x amount of time, but we also don't stream super small data packets. + # So, the right way to do this is with response.raw.stream(). + # This will read each chunk as it comes in, and return each complete chunk. This is the same way a web browser will handle the data, where it won't handle the data until the entire + # chunk is read. So there's no need to stream sub-chunks. # - # Note this logic will always have a "one message" latency due to the way we get blocked on the socket. Even though we read small amounts, there's no way to know the full message - # length, and thus there's no way to ask for only the remainder of the message. Thus, the end of the message will usually always get put into a pending read, but that read will block - # until the buffer is filled the rest of the way with the n+1 message. Unfortunately that means we are always a message or so behind in the stream. Without being able to do a non-blocking request read, - # there's no way to work around this. + # There's still a problem with stream, which is it will block until the next chunk is ready. But for us, once we have data, we want to send it. Thus we must spin off a thread to do + # the stream reading, and then transfer the buffer. Also stream() is a generator, so it can't be re-entered. # - # Ideally if we could just peak at the pending data length without blocking, we could do this much more efficiently. - def doUnknownBodySizeRead(self, octoHttpResult:OctoHttpRequest.Result): - - # How much we will micro read, this needs to be quite small, to prevent getting "stuck" between messages. - microReadSizeBytes = 300 + def doUnknownBodyChunkRead(self, httpResult:OctoHttpRequest.Result): + + # Even though we read complete chunks as they come in, we might want to buffer smaller chunks up + # before sending them so the compression and stream is more efficient. + # This does need to be small, because we wan't reading this min time period back to back, + # we are reading a chunk, doing all of the send logic, and then spinning back to here. + # So if we set this at exactly 16.6 for a 60fps stream, for example, we will fall behind. + minBufferBuildTimeSec = 0.010 # 10ms + + # Just as a sanity check, we will define the max amount of time we will wait for one chunk. + # This will make sure we don't get stuck in a loop if there are any bugs. + maxChunkReadTimeSec = 20 * 60 * 60 # 20 hours + + # If this is the first time, setup the unknown body read info. + # Once this is defined, this body read method must be used for the rest of the request. + if self.UnknownBodyChunkReadContext is None: + context = UnknownBodyChunkReadContext(httpResult) + context.Thread = threading.Thread(target=self.doUnknownBodyChunkReadThread) + context.Thread.start() + self.UnknownBodyChunkReadContext = context - # How long we will build one big buffer before returning, in seconds. - # Note the first read will get double this time. - maxBufferBuildTimeSec = 0.050 #(50ms) - - # Vars - buildReads = 0 - buffer = bytearray(2 * 1024) - bufferSize = 0 try: startSec = time.time() - while True: - # Do a small read, which will block until the full (small) size is read. - # If nothing shows up to be read, this will wait until the http request read timeout expires, and then will return None. - currentReadBuffer = self.doBodyRead(octoHttpResult, microReadSizeBytes) + chunkBufferList = None + + # Since we will always sleep for at least the min time, there's no need to do work until the min time is meet. + # If we did do the loop, we would just end up spinning and sleeping again. + time.sleep(minBufferBuildTimeSec) + + # Try to read a chunk or wait for the read to be done. + # Only try to read while the stream is open. + while self.IsClosed is False: + + # First, sanity check we haven't been running forever. + now = time.time() + if now - startSec > maxChunkReadTimeSec: + raise Exception(f"doUnknownBodyChunkRead has been waiting for a chunk for {maxChunkReadTimeSec} seconds. This is an error.") + + # Next, check if there are any new buffers to read. + with self.UnknownBodyChunkReadContext.BufferLock: + if len(self.UnknownBodyChunkReadContext.BufferList) > 0: + # If there's new chunks, grab them all and reset the buffer list. + if chunkBufferList is None: + chunkBufferList = self.UnknownBodyChunkReadContext.BufferList + else: + chunkBufferList += self.UnknownBodyChunkReadContext.BufferList + self.UnknownBodyChunkReadContext.BufferList = [] + # Clear the event under lock, so we don't miss a new set. + self.UnknownBodyChunkReadContext.BufferDataReadyEvent.clear() + + # If we got some chunks, see if we are past the min chunk read time or if the chunk stream is complete. + if chunkBufferList is not None and now - startSec > minBufferBuildTimeSec: + break - # If None is returned, we are done. Return the current buffer or None. - if currentReadBuffer is None: + # Finally, AFTER we checked if we have new buffers, check is the read is done. + # Note we have to do this after we grab any new buffers in the list, because we can have pending chunks from before the stream is closed. + if self.UnknownBodyChunkReadContext.ReadComplete: break - # Copy into the existing buffer. - buffer[bufferSize:bufferSize+len(currentReadBuffer)] = currentReadBuffer - bufferSize += len(currentReadBuffer) - buildReads += 1 + # If we don't have a chunk, wait on the event until we have something. + # This will return when there's new chunks ready, ReadComplete is set, or it hits a timeout. + self.UnknownBodyChunkReadContext.BufferDataReadyEvent.wait(maxChunkReadTimeSec) - # We have noticed for some systems (like OctoFarm) the first read takes a while for the event stream to get - # going, and then it gets data. The problem here is we unblock with the first chunk of data and then we are at our time - # limit and return. Instead, it's more ideal to allow one more time limit so we can read the full message and then return it. - if self.IsFirstMicroBodyRead: - self.IsFirstMicroBodyRead = False - startSec = time.time() + # If we broke out of the loop and we have no chunks to send, we are done. + if chunkBufferList is None: + return None - # Check if it's time to be done. - if time.time() - startSec > maxBufferBuildTimeSec: - break + # Append all of the chunks together and return the buffer! + # Optimize for the single chunk scenario. + if len(chunkBufferList) == 1: + return chunkBufferList[0] - # If we broke out, it's time to return what we have. - # If we didn't read anything, we want to return none, to indicate we are done or there was a read timeout. - if bufferSize == 0: - return None + # Find the final buffer length. + totalLength = sum(len(b) for b in chunkBufferList) + + # Allocate a buffer to hold all of the chunks. + finalBuffer = bytearray(totalLength) + offset = 0 + for buffer in chunkBufferList: + view = memoryview(buffer) + with view: + finalBuffer[offset:offset + len(view)] = view + offset += len(view) - # Return the subset of the buffer we filled. - return buffer[0:bufferSize] + # Sanity check + if len(finalBuffer) != totalLength: + raise Exception(f"Final appended buffer was {len(finalBuffer)} but it should have been {totalLength}") + + # Return! + return finalBuffer except Exception as e: Sentry.Exception(self.getLogMsgPrefix()+ " exception thrown in doUnknownBodySizeRead. Ending body read.", e) @@ -1174,7 +1267,12 @@ def doUnknownBodySizeRead(self, octoHttpResult:OctoHttpRequest.Result): # Based on the content length and the content type, determine if we should do a doUnknownBodySizeRead read. # Read doUnknownBodySizeRead about why we need to use it, but since it's not efficient, we only want to use it when we know we should. - def shouldDoUnknownBodySizeRead(self, contentTypeLower_CanBeNone, contentLengthLower_CanBeNone): + def shouldDoUnknownBodyChunkRead(self, contentTypeLower_CanBeNone:str, contentLengthLower_CanBeNone:int): + + # If this is set, we are already doing a unknown body chunk read, so we must keep doing it. + if self.UnknownBodyChunkReadContext is not None: + return True + # If there's a known content length, there's no need to do this, because the normal read will fill the requested buffer size # but return the remainder subset immediately when the full buffer is read. if contentLengthLower_CanBeNone is not None: @@ -1207,3 +1305,20 @@ def checkForDelayIfNotHighPri(self): # Formatting helper. def _FormatFloat(self, value:float) -> str: return str(format(value, '.3f')) + + +# Used to capture the context of the unknown body read thread. +class UnknownBodyChunkReadContext: + + def __init__(self, httpResult:OctoHttpRequest.Result) -> None: + self.HttpResult = httpResult + self.Thread:threading.Thread = None + self.BufferLock = threading.Lock() + self.BufferDataReadyEvent = threading.Event() + + # We use a list so we can efficiently append all of the pending buffers at once when they are being sent. + self.BufferList = [] + + # Set to true when the read is done either from the end of the body or an error. + # Once true, it will never read again, but we do need to process the BufferList + self.ReadComplete = False diff --git a/octoeverywhere/octohttprequest.py b/octoeverywhere/octohttprequest.py index 6c3bc9c..6ef3685 100644 --- a/octoeverywhere/octohttprequest.py +++ b/octoeverywhere/octohttprequest.py @@ -433,6 +433,8 @@ def MakeHttpCallAttempt(logger, attemptName, method, url, headers, data, mainRes # # Note we use a long timeout because some api calls can hang for a while. # For example when plugins are installed, some have to compile which can take some time. + # timeout note! This value also effects how long a body read can be. This can effect unknown body chunk stream reads can hang while waiting on a chunk. + # But whatever this timeout value is will be the max time a body read can take, and then the chunk will fail and the stream will close. # # See the note about allowRedirects above MakeHttpCall. # diff --git a/setup.py b/setup.py index 57b6356..697c68f 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this single version string is used by all of the plugins in OctoEverywhere! -plugin_version = "3.6.3" +plugin_version = "3.6.4" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 83ccdb286af6026400d7198a1f9322dd414b1808 Mon Sep 17 00:00:00 2001 From: Patrick Irish Date: Sun, 8 Dec 2024 01:33:53 -0500 Subject: [PATCH 163/328] Update dockerfile to build and run as non root user (#85) --- Dockerfile | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index aae9e4d..8987d34 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,14 +2,15 @@ # since we need some advance binaries for things like pillow and ffmpeg. FROM alpine:3.20.0 -# We will base ourselves in root, becuase why not. -WORKDIR /root +RUN adduser -Ss /bin/bash app -h /app -g root -u 1001 + +WORKDIR /app # Define some user vars we will use for the image. # These are read in the docker_octoeverywhere module, so they must not change! -ENV USER=root -ENV REPO_DIR=/root/octoeverywhere -ENV VENV_DIR=/root/octoeverywhere-env +ENV USER=app +ENV REPO_DIR=/app/octoeverywhere +ENV VENV_DIR=/app/octoeverywhere-env # This is a special dir that the user MUST mount to the host, so that the data is persisted. # If this is not mounted, the printer will need to be re-linked everytime the container is remade. ENV DATA_DIR=/data/ @@ -36,7 +37,10 @@ RUN ${VENV_DIR}/bin/pip3 install --require-virtualenv --no-cache-dir -q -r ${REP RUN apk add zstd RUN ${VENV_DIR}/bin/pip3 install --require-virtualenv --no-cache-dir -q "zstandard>=0.21.0,<0.23.0" +# Ensure directories have correct ownership. Having the group set to root(0) and writable by group will allow this to run on openshift +RUN chown -R 1001:0 /app && chmod -R g+wx /app + # For docker, we use our docker_octoeverywhere host to handle the runtime setup and launch of the serivce. WORKDIR ${REPO_DIR} # Use the full path to the venv, we msut use this [] notation for our ctlc handler to work in the contianer -ENTRYPOINT ["/root/octoeverywhere-env/bin/python", "-m", "docker_octoeverywhere"] +ENTRYPOINT ["/app/octoeverywhere-env/bin/python", "-m", "docker_octoeverywhere"] From 98dc8ae36c3ca1673aa810922fb0a4586381e3fe Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sat, 7 Dec 2024 22:36:09 -0800 Subject: [PATCH 164/328] Minor edits to the dockerfile. --- Dockerfile | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8987d34..806ffac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,9 @@ -# Start with the lastest alpine, for a solid base, +# Start with the latest alpine, for a solid base, # since we need some advance binaries for things like pillow and ffmpeg. FROM alpine:3.20.0 +# Create a non-root user to run, so we don't run as root. +# There's no need to run as root and it helps some platforms like openshift. RUN adduser -Ss /bin/bash app -h /app -g root -u 1001 WORKDIR /app @@ -12,27 +14,27 @@ ENV USER=app ENV REPO_DIR=/app/octoeverywhere ENV VENV_DIR=/app/octoeverywhere-env # This is a special dir that the user MUST mount to the host, so that the data is persisted. -# If this is not mounted, the printer will need to be re-linked everytime the container is remade. +# If this is not mounted, the printer will need to be re-linked every time the container is remade. ENV DATA_DIR=/data/ # Install the required packages. -# Any packages here should be mirrored in the install script - and any optaionl pillow packages done inline. +# Any packages here should be mirrored in the install script - and any optional pillow packages done inline. # GCC, python3-dev, and musl-dev are required for pillow, and jpeg-dev and zlib-dev are required for jpeg support. RUN apk add --no-cache curl ffmpeg jq python3 python3-dev gcc musl-dev py3-pip py3-virtualenv jpeg-dev libjpeg-turbo-dev zlib-dev py3-pillow libffi-dev # -# We decided to not run the installer, since the point of the installer is to setup the env, build the launch args, and setup the serivce. -# Instead, we will manually run the smaller subset of commands that are requred to get the env setup in docker. +# We decided to not run the installer, since the point of the installer is to setup the env, build the launch args, and setup the service. +# Instead, we will manually run the smaller subset of commands that are required to get the env setup in docker. # Note that if this ever becomes too much of a hassle, we might want to revert back to using the installer, and supporting a headless install. # RUN virtualenv -p /usr/bin/python3 ${VENV_DIR} RUN ${VENV_DIR}/bin/python -m pip install --upgrade pip -# Copy the entire repo into the image, do this as late as possible to avoid rebuilding the image everytime the repo changes. +# Copy the entire repo into the image, do this as late as possible to avoid rebuilding the image every time the repo changes. COPY ./ ${REPO_DIR}/ RUN ${VENV_DIR}/bin/pip3 install --require-virtualenv --no-cache-dir -q -r ${REPO_DIR}/requirements.txt -# Install the optional pacakges for zstandard compression. +# Install the optional packages for zstandard compression. # THIS VERSION STRING MUST STAY IN SYNC with Compression.ZStandardPipPackageString RUN apk add zstd RUN ${VENV_DIR}/bin/pip3 install --require-virtualenv --no-cache-dir -q "zstandard>=0.21.0,<0.23.0" @@ -40,7 +42,8 @@ RUN ${VENV_DIR}/bin/pip3 install --require-virtualenv --no-cache-dir -q "zstanda # Ensure directories have correct ownership. Having the group set to root(0) and writable by group will allow this to run on openshift RUN chown -R 1001:0 /app && chmod -R g+wx /app -# For docker, we use our docker_octoeverywhere host to handle the runtime setup and launch of the serivce. +# For docker, we use our docker_octoeverywhere host to handle the runtime setup and launch of the service. WORKDIR ${REPO_DIR} -# Use the full path to the venv, we msut use this [] notation for our ctlc handler to work in the contianer + +# Use the full path to the venv, we must use this [] notation for our ctl-c handler to work in the container. ENTRYPOINT ["/app/octoeverywhere-env/bin/python", "-m", "docker_octoeverywhere"] From 12a0f1302ee588f5a0c7e0c667a608eb6ef081ff Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Thu, 19 Dec 2024 14:32:13 -0800 Subject: [PATCH 165/328] Updating the bambu docker to make local mode more clear and perfered. --- .vscode/launch.json | 22 +++- bambu_octoeverywhere/bambuclient.py | 35 +++--- bambu_octoeverywhere/bambucloud.py | 3 +- docker-compose.yml | 24 +++- docker_octoeverywhere/__main__.py | 177 ++++++++++++++++++++-------- linux_host/config.py | 6 + octoeverywhere/sentry.py | 121 +++++++++---------- py_installer/ConfigHelper.py | 2 + 8 files changed, 262 insertions(+), 128 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index c54e0bb..adb136e 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -36,7 +36,7 @@ ] }, { - "name": "Bambu Connect - P1S", + "name": "Bambu Connect - X1C", "type": "debugpy", "request": "launch", "module": "bambu_octoeverywhere", @@ -51,7 +51,7 @@ ] }, { - "name": "Bambu Connect - X1C", + "name": "Bambu Connect - P1S", "type": "debugpy", "request": "launch", "module": "bambu_octoeverywhere", @@ -129,5 +129,23 @@ "module": "octoprint_octoeverywhere", "justMyCode": true }, + { + "name": "Docker Container Bootstrap & Run", + "type": "debugpy", + "request": "launch", + "module": "docker_octoeverywhere", + "justMyCode": false, + "env": { + "VENV_DIR": "/home/pi/octoeverywhere-env", + "REPO_DIR": "/home/pi/octoeverywhere", + "DATA_DIR": "/home/pi/.octoeverywhere-docker-bootstrap", // This path needs to be created. + //"SERIAL_NUMBER": "serial", + //"ACCESS_CODE": "test123" + //"PRINTER_IP": "127.0.0.1", + //"CONNECTION_MODE": "cloud", + //"BAMBU_CLOUD_ACCOUNT_EMAIL":"quinn@test.com", + //"BAMBU_CLOUD_ACCOUNT_PASSWORD": "test123", + }, + }, ] } \ No newline at end of file diff --git a/bambu_octoeverywhere/bambuclient.py b/bambu_octoeverywhere/bambuclient.py index a86fad0..bb896d5 100644 --- a/bambu_octoeverywhere/bambuclient.py +++ b/bambu_octoeverywhere/bambuclient.py @@ -133,8 +133,8 @@ def _ClientWorker(self): # We are connecting to Bambu Cloud, setup MQTT for it. self.Client.tls_set(tls_version=ssl.PROTOCOL_TLS) else: - # We are trying to connect to the printer locally, so configure mqtt for a LAN connection. - self.Logger.info("Trying to connect to printer via LAN...") + # We are trying to connect to the printer locally, so configure mqtt for a local connection. + self.Logger.info("Trying to connect to printer via local connection...") self.Client.tls_set(tls_version=ssl.PROTOCOL_TLS, cert_reqs=ssl.CERT_NONE) self.Client.tls_insecure_set(True) @@ -380,20 +380,25 @@ def _GetConnectionContextToTry(self) -> ConnectionContext: if self.ConsecutivelyFailedConnectionAttempts > 6: self.ConsecutivelyFailedConnectionAttempts = 0 + # Get the connection mode set by the user. This defaults to local, but the user can explicitly set it to either. + connectionMode = self.Config.GetStr(Config.SectionBambu, Config.BambuConnectionMode, Config.BambuConnectionModeDefault) + if connectionMode == Config.BambuConnectionModeValueCloud: + # If the mode is set to cloud, try to connect via it. + # If a context can't be created, there's something wrong with the account info + # or a Bambu service issue. Since we have the local info, we can try it as well. + cloudContext = self._TryToGetCloudConnectContext() + if cloudContext is not None: + return cloudContext + self.Logger.warning("We tried to connect via Bambu Cloud, but failed. We will try a local connection.") + # On the first few attempts, use the expected IP or the cloud config. # The first attempt will always be attempt 1, since it's reset to 0 and incremented before connecting. # The IP can be empty, like if the docker container is used, in which case we should always search for the printer. configIpOrHostname = self.Config.GetStr(Config.SectionCompanion, Config.CompanionKeyIpOrHostname, None) if self.ConsecutivelyFailedConnectionAttempts < 4: - # If we are using a Bambu cloud connection, try to return a connection object for it. - # We always try to do this for the first few attempts, since if it's setup as a Cloud connection, a LAN connection most likely won't work. - cloudContext = self._TryToGetCloudConnectContext() - if cloudContext is not None: - return cloudContext - - # If we aren't using a cloud connection or it failed, return the LAN hostname + # If we aren't using a cloud connection or it failed, return the local hostname if configIpOrHostname is not None and len(configIpOrHostname) > 0: - return self._GetLanConnectionContext(configIpOrHostname) + return self._GetLocalConnectionContext(configIpOrHostname) # If we fail too many times, try to scan for the printer on the local subnet, the IP could have changed. # Since we 100% identify the printer by the access token and printer SN, we can try to scan for it. @@ -408,13 +413,13 @@ def _GetConnectionContextToTry(self) -> ConnectionContext: ip = ips[0] self.Logger.info(f"We found a new IP for this printer. [{configIpOrHostname} -> {ip}] Updating the config and using it to connect.") self.Config.SetStr(Config.SectionCompanion, Config.CompanionKeyIpOrHostname, ip) - return self._GetLanConnectionContext(ip) + return self._GetLocalConnectionContext(ip) # If we don't find anything, just use the config IP. - return self._GetLanConnectionContext(configIpOrHostname) + return self._GetLocalConnectionContext(configIpOrHostname) - def _GetLanConnectionContext(self, ipOrHostname) -> ConnectionContext: + def _GetLocalConnectionContext(self, ipOrHostname) -> ConnectionContext: # The username is always the same, we use the local LAN access token. return ConnectionContext(False, ipOrHostname, "bblp", self.LanAccessCode) @@ -436,7 +441,9 @@ def _TryToGetCloudConnectContext(self) -> ConnectionContext: if accessTokenResult.Status == LoginStatus.BadUserNameOrPassword: self.Logger.error("The email address or password is wrong. Re-run the Bambu Connect installer or use the docker files to update your email address and password.") elif accessTokenResult.Status == LoginStatus.TwoFactorAuthEnabled: - self.Logger.error("To factor auth is enabled on this account. Bambu Lab doesn't allow us to support two factor auth, so it must be disabled on your account or LAN Only mode must be used on the printer.") + self.Logger.error("Two factor auth is enabled on this account. Bambu Lab doesn't allow us to support two factor auth, so it must be disabled on your account or the local connection mode.") + elif accessTokenResult.Status == LoginStatus.EmailCodeRequired: + self.Logger.error("This account requires an email code to login. Bambu Lab doesn't allow us to support this, so you must use the local connection mode.") else: self.Logger.error("Unknown error, we will try again later.") self.Logger.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") diff --git a/bambu_octoeverywhere/bambucloud.py b/bambu_octoeverywhere/bambucloud.py index 461504b..43ab47f 100644 --- a/bambu_octoeverywhere/bambucloud.py +++ b/bambu_octoeverywhere/bambucloud.py @@ -17,7 +17,8 @@ class LoginStatus(Enum): Success = 0 # This is the only successful value TwoFactorAuthEnabled = 1 BadUserNameOrPassword = 2 - UnknownError = 3 + EmailCodeRequired = 3 + UnknownError = 4 # The result of a get access token request. diff --git a/docker-compose.yml b/docker-compose.yml index b2a8f3b..be4eac8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,8 +9,19 @@ services: - SERIAL_NUMBER=XXXXXXXXXXXXXXX # Find using the printer's display or use https://octoeverywhere.com/s/bambu-ip - PRINTER_IP=XXX.XXX.XXX.XXX + + # Optionally: If you want to connect via the Bambu Cloud, you can specify the following environment variables. + # By default the plugin will use the local connection mode, which is preferred. + # Bambu Cloud might not work for all printers and account types due to limitations by Bambu Labs. :( + # + # If you use Bambu Cloud, you MUST disable 2 factor authentication, because Bambu does not allow us to support it. + # Your Bambu email address and password are KEPT LOCALLY, securely on disk, and are NEVER SENT to the OctoEverywhere service + # - BAMBU_CLOUD_ACCOUNT_EMAIL=quinn@test.com + # - BAMBU_CLOUD_ACCOUNT_PASSWORD=supersecretpassword + # - CONNECTION_MODE=cloud + volumes: - # This can also be an absolue path, e.g. /var/octoeverywhere/plugin/data or /c/users/name/plugin/data + # This can also be an absolute path, e.g. /var/octoeverywhere/plugin/data or /c/users/name/plugin/data - ./data:/data # Add as many printers as you want! @@ -23,6 +34,17 @@ services: # - SERIAL_NUMBER=XXXXXXXXXXXXXXX # # Find using the printer's display or use https://octoeverywhere.com/s/bambu-ip # - PRINTER_IP=XXX.XXX.XXX.XXX + # + # # Optionally: If you want to connect via the Bambu Cloud, you can specify the following environment variables. + # # By default the plugin will use the local connection mode, which is preferred. + # # Bambu Cloud might not work for all printers and account types due to limitations by Bambu Labs. :( + # # + # # If you use Bambu Cloud, you MUST disable 2 factor authentication, because Bambu does not allow us to support it. + # # Your Bambu email address and password are KEPT LOCALLY, securely on disk, and are NEVER SENT to the OctoEverywhere service + # # - BAMBU_CLOUD_ACCOUNT_EMAIL=quinn@test.com + # # - BAMBU_CLOUD_ACCOUNT_PASSWORD=supersecretpassword + # # - CONNECTION_MODE=cloud + # # volumes: # # Specify a path mapping for the required persistent storage # - ./data:/data \ No newline at end of file diff --git a/docker_octoeverywhere/__main__.py b/docker_octoeverywhere/__main__.py index 7b7b7e0..e50a943 100644 --- a/docker_octoeverywhere/__main__.py +++ b/docker_octoeverywhere/__main__.py @@ -73,7 +73,13 @@ def CreateDirIfNotExists(path: str) -> None: config = Config(configPath) - # The serial number is always required, in both Bambu Cloud and LAN mode. + # + # + # Step 1: Ensure all required vars are set. + # + # + + # The serial number is always required, in both Bambu Cloud and local connection mode. # So we always get that first. printerSn = os.environ.get("SERIAL_NUMBER", None) if printerSn is not None: @@ -84,7 +90,7 @@ def CreateDirIfNotExists(path: str) -> None: logger.error("") logger.error("") logger.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") - logger.error(" You must pass the printer's Serial Number as an env var.") + logger.error(" You must provide your printer's Serial Number.") logger.error("Use `docker run -e SERIAL_NUMBER=` or add it to your docker-compose file.") logger.error("") logger.error(" To find your Serial Number -> https://octoeverywhere.com/s/bambu-sn") @@ -95,39 +101,137 @@ def CreateDirIfNotExists(path: str) -> None: time.sleep(5.0) sys.exit(1) + # The access code is also always required, in both Bambu Cloud and local connection mode. + # In cloud mode the we can get the access code from the service, but it's often wrong...? + # If there is a arg passed, always update or set it. + # This allows users to update the values after the image has ran the first time. + accessCode = os.environ.get("ACCESS_CODE", None) + if accessCode is not None: + logger.info(f"Setting Access Code: {accessCode}") + config.SetStr(Config.SectionBambu, Config.BambuAccessToken, accessCode) + # Ensure something is set now. + if config.GetStr(Config.SectionBambu, Config.BambuAccessToken, None) is None: + logger.error("") + logger.error("") + logger.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + logger.error(" You must provide your printer's Access Code.") + logger.error(" Use `docker run -e ACCESS_CODE=` or add it to your docker-compose file.") + logger.error("") + logger.error(" To find your Access Code -> https://octoeverywhere.com/s/access-code") + logger.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + logger.error("") + logger.error("") + # Sleep some, so we don't restart super fast and then exit. + time.sleep(5.0) + sys.exit(1) + + # For now, we also need the user to supply the printer's IP address in both modes, since we can't auto scan the network in docker. + # We also need this for the Bambu Cloud mode, since we can't get it from the Bambu Cloud API and we can't scan for the printer. + printerId = os.environ.get("PRINTER_IP", None) + if printerId is not None: + logger.info(f"Setting Printer IP: {printerId}") + config.SetStr(Config.SectionCompanion, Config.CompanionKeyIpOrHostname, printerId) + # Ensure something is set now. + if config.GetStr(Config.SectionCompanion, Config.CompanionKeyIpOrHostname, None) is None: + logger.error("") + logger.error("") + logger.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + logger.error(" You must provide your printer's IP Address.") + logger.error(" Use `docker run -e PRINTER_IP=` or add it to your docker-compose file.") + logger.error("") + logger.error(" To find your printer's IP Address -> https://octoeverywhere.com/s/bambu-ip") + logger.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + logger.error("") + logger.error("") + # Sleep some, so we don't restart super fast and then exit. + time.sleep(5.0) + sys.exit(1) + + # The port is always the same, so we just set the known Bambu Lab printer port. + if config.GetStr(Config.SectionCompanion, Config.CompanionKeyPort, None) is None: + config.SetStr(Config.SectionCompanion, Config.CompanionKeyPort, "8883") + + # + # + # Step 2: Determine the connection mode. Bambu Cloud or Local. + # + # # Bambu updated the printer and broke LAN access unless the printer is in LAN mode. # The work around was to connect to the Bambu Cloud instead of directly to the printer. # The biggest downside of this is that we need to get the user's email address and password for Bambu Cloud. # BUT the user can also do the LAN only mode, if they want to. - isLanOnlyMode = bool(os.environ.get("LAN_ONLY_MODE", "").lower() in ("true", "1", "yes")) - isAccessCodeRequired = True - if isLanOnlyMode: - # In LAN only mode we only need the Serial number and access code. - logger.info("Connection Mode: LAN Only (Use the env var LAN_ONLY_MODE=FALSE to enable Bambu Cloud mode.)") - # This is LAN only mode, where we need the user to get us the Access Code. (In the cloud mode, we can get it from the Bambu Cloud API) - # If there is a arg passed, always update or set it. - # This allows users to update the values after the image has ran the first time. - accessCode = os.environ.get("ACCESS_CODE", None) - if accessCode is not None: - logger.info(f"Setting Access Code: {accessCode}") - config.SetStr(Config.SectionBambu, Config.BambuAccessToken, accessCode) - # Ensure something is set now. - if config.GetStr(Config.SectionBambu, Config.BambuAccessToken, None) is None: + # + # It seems that bambu may have walked back on the no LAN access, so for now we default to the local mode, which is preferred. + useBambuCloud = False + envVarBambuCloudEmailKey = "BAMBU_CLOUD_ACCOUNT_EMAIL" + envVarBambuCloudPasswordKey = "BAMBU_CLOUD_ACCOUNT_PASSWORD" + envVarConnectionModeKey = "CONNECTION_MODE" + + # First, we will see if the user explicitly set the mode. + connectionModeVar = os.environ.get(envVarConnectionModeKey, None) + if connectionModeVar is not None: + # Ensure the passed value is what we expect. + connectionModeVar = connectionModeVar.lower().strip() + if connectionModeVar == "cloud" or connectionModeVar == "bambucloud": + useBambuCloud = True + elif connectionModeVar == "local": + useBambuCloud = False + else: logger.error("") logger.error("") logger.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") - logger.error(" You must pass the printer's Access Code as an env var.") - logger.error(" Use `docker run -e ACCESS_CODE=` or add it to your docker-compose file.") + logger.error(" Invalid CONNECTION_MODE value passed.") + logger.error("") + logger.error(" To connect via the Bambu Cloud use the value of `cloud`.") + logger.error(" To make a local connection via your local network use the value of `local`.") + logger.error("") + logger.error(" Use `docker run -e CONNECTION_MODE=` or add it to your docker-compose file.") logger.error("") - logger.error(" To find your Access Code -> https://octoeverywhere.com/s/access-code") logger.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") logger.error("") logger.error("") # Sleep some, so we don't restart super fast and then exit. time.sleep(5.0) sys.exit(1) + logger.info(f"Connection mode specified as: {connectionModeVar}. Use Cloud Connection Mode: {useBambuCloud}") else: - logger.info("Connection Mode: Bambu Cloud (Use the env var LAN_ONLY_MODE=TRUE to enable LAN Only mode.)") + # If the user didn't explicitly pick a mode, we will figure it out based on what they passed. + # This is a legacy flag, check if first. + lanOnlyMode = os.environ.get("LAN_ONLY_MODE", None) + if lanOnlyMode is not None: + useBambuCloud = not bool(lanOnlyMode.lower().strip() in ("true", "1", "yes")) + logger.info(f"LAN_ONLY_MODE specified as: {lanOnlyMode}. Use Cloud Connection Mode: {useBambuCloud}") + else: + # If there are no explicit flags, check if it's set in the config. + configMode = config.GetStr(Config.SectionBambu, Config.BambuConnectionMode, None) + if configMode is not None: + if configMode == "cloud": + useBambuCloud = True + elif configMode == "local": + useBambuCloud = False + else: + logger.error(f"Invalid Bambu Connection Mode in config: {configMode}, defaulting to local.") + useBambuCloud = False + logger.info(f"Connection mode found in the OctoEverywhere config as `{configMode}`.") + else: + # There's nothing passed and nothing in the config, figure it out based on if they user passed the email and password. + if os.environ.get(envVarBambuCloudEmailKey, None) is not None or os.environ.get(envVarBambuCloudPasswordKey, None) is not None: + useBambuCloud = True + logger.info("Bambu cloud email or password supplied, so we will use Bambu Cloud connection mode.") + else: + useBambuCloud = False + logger.info("No bambu cloud email or password passed, so we will use the preferred local connection mode.") + logger.info(f"If you want to change the connection mode, use `{envVarConnectionModeKey}` which can be set to 'cloud' or 'local'.") + + # Explicitly set the mode in the config. + config.SetStr(Config.SectionBambu, Config.BambuConnectionMode, "cloud" if useBambuCloud else "local") + + # + # + # Step 3: Get any required Bambu Cloud values. + # + # + if useBambuCloud: # In Bambu Cloud mode, we need the user's email and password. bambuCloud = BambuCloud(logger, config) # Get any existing values. @@ -140,7 +244,7 @@ def CreateDirIfNotExists(path: str) -> None: logger.error("") logger.error("") logger.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") - logger.error(" You must pass your Bambu Cloud account email address as an env var.") + logger.error(" You must provide your Bambu Cloud account email address.") logger.error("Use `docker run -e BAMBU_CLOUD_ACCOUNT_EMAIL=` or add it to your docker-compose file.") logger.error("") logger.error(" Your Bambu email address and password are KEPT LOCALLY, encrypted on disk") @@ -157,7 +261,7 @@ def CreateDirIfNotExists(path: str) -> None: logger.error("") logger.error("") logger.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") - logger.error(" You must pass your Bambu Cloud account password as an env var.") + logger.error(" You must provide your Bambu Cloud account password.") logger.error("Use `docker run -e BAMBU_CLOUD_ACCOUNT_PASSWORD=` or add it to your docker-compose file.") logger.error("") logger.error(" Your Bambu email address and password are KEPT LOCALLY, encrypted on disk") @@ -190,35 +294,6 @@ def CreateDirIfNotExists(path: str) -> None: logger.info("Setting Bambu Cloud to the default value for world wide accounts.") config.SetStr(Config.SectionBambu, Config.BambuCloudRegion, "worldwide") - # For now, we also need the user to supply the printer's IP address, since we can't auto scan the network in docker. - # We also need this for the Bambu Cloud mode, since we can't get it from the Bambu Cloud API and we can't scan for the printer. - printerId = os.environ.get("PRINTER_IP", None) - if printerId is not None: - logger.info(f"Setting Printer IP: {printerId}") - config.SetStr(Config.SectionCompanion, Config.CompanionKeyIpOrHostname, printerId) - # Ensure something is set now. - if config.GetStr(Config.SectionCompanion, Config.CompanionKeyIpOrHostname, None) is None: - logger.error("") - logger.error("") - logger.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") - logger.error(" You must pass the printer's IP Address as an env var.") - logger.error(" Use `docker run -e PRINTER_IP=` or add it to your docker-compose file.") - logger.error("") - logger.error(" To find your printer's IP Address -> https://octoeverywhere.com/s/bambu-ip") - logger.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") - logger.error("") - logger.error("") - # Sleep some, so we don't restart super fast and then exit. - time.sleep(5.0) - sys.exit(1) - - # The port is always the same, so we just set the known Bambu Lab printer port. - if config.GetStr(Config.SectionCompanion, Config.CompanionKeyPort, None) is None: - config.SetStr(Config.SectionCompanion, Config.CompanionKeyPort, "8883") - - # We don't set the IP address of the printer. The Bambu Connect plugin will automatically find the printer - # on the local network using the Access Token and SN. By not setting the value, it will force it to search first. - # Create the rest of the required dirs based in the data dir, since it's persistent. localStoragePath = os.path.join(dataPath, "octoeverywhere-store") CreateDirIfNotExists(localStoragePath) diff --git a/linux_host/config.py b/linux_host/config.py index 2a9df2a..7e2342b 100644 --- a/linux_host/config.py +++ b/linux_host/config.py @@ -63,6 +63,11 @@ class Config: # Used if the user is logged into Bambu Cloud BambuCloudContext = "cloud_context" BambuCloudRegion = "cloud_region" + # Explicitly defines what connection mode we are using. Can be "cloud" or "local". Defaults to local + BambuConnectionMode = "connection_mode" + BambuConnectionModeValueLocal = "local" + BambuConnectionModeValueCloud = "cloud" + BambuConnectionModeDefault = BambuConnectionModeValueLocal # This allows us to add comments into our config. @@ -76,6 +81,7 @@ class Config: { "Target": CompanionKeyPort, "Comment": "The port this companion plugin will use to connect to Moonraker. The OctoEverywhere plugin service needs to be restarted before changes will take effect."}, { "Target": BambuAccessToken, "Comment": "The access token to the Bambu printer. It can be found using the LCD screen on the printer, in the settings. The OctoEverywhere plugin service needs to be restarted before changes will take effect."}, { "Target": BambuPrinterSn, "Comment": "The serial number of your Bambu printer. It can be found using this guide: https://wiki.bambulab.com/en/general/find-sn The OctoEverywhere plugin service needs to be restarted before changes will take effect."}, + { "Target": BambuConnectionMode,"Comment": "The connection mode used for Bambu Connect. Can be 'cloud' or 'local'. 'cloud' will use the bambu cloud which requires the user's email and password to be set, `local` will connect via the LAN."}, { "Target": WebcamNameToUseAsPrimary, "Comment": "This is the webcam name OctoEverywhere will use for Gadget AI, notifications, and such. This much match the camera 'Name' from your Mainsail of Fluidd webcam settings. The default value of 'Default' will pick whatever camera the system can find."}, { "Target": WebcamAutoSettings, "Comment": "Enables or disables auto webcam setting detection. If enabled, OctoEverywhere will find the webcam settings configured via the frontend (Fluidd, Mainsail, etc) and use them. Disable to manually set the values and have them not be overwritten."}, { "Target": WebcamStreamUrl, "Comment": "Webcam streaming URL. This can be a local relative path (ex: /webcam/?action=stream) or absolute http URL (ex: http://10.0.0.1:8080/webcam/?action=stream or http://webcam.local/webcam/?action=stream)"}, diff --git a/octoeverywhere/sentry.py b/octoeverywhere/sentry.py index 58a6871..d67c7ce 100644 --- a/octoeverywhere/sentry.py +++ b/octoeverywhere/sentry.py @@ -3,10 +3,10 @@ import time import traceback -import sentry_sdk -from sentry_sdk import Hub -from sentry_sdk.integrations.logging import LoggingIntegration -from sentry_sdk.integrations.threading import ThreadingIntegration +# import sentry_sdk +# from sentry_sdk import Hub +# from sentry_sdk.integrations.logging import LoggingIntegration +# from sentry_sdk.integrations.threading import ThreadingIntegration from .exceptions import NoSentryReportException from .threaddebug import ThreadDebug @@ -42,44 +42,46 @@ def Setup(versionString:str, distType:str, isDevMode:bool = False, enableProfili Sentry.RestartProcessOnCantCreateThreadBug = restartOnCantCreateThreadBug # Only setup sentry if we aren't in dev mode. - if Sentry.IsDevMode is False: - try: - # We don't want sentry to capture error logs, which is it's default. - # We do want the logging for breadcrumbs, so we will leave it enabled. - sentry_logging = LoggingIntegration( - level=logging.INFO, # Capture info and above as breadcrumbs - event_level=logging.FATAL # Only send FATAL errors and above. - ) - # Setup and init - sentry_sdk.init( - dsn= "https://883879bfa2402df86c098f6527f96bfa@oe-sentry.octoeverywhere.com/4", - integrations= [ - sentry_logging, - ThreadingIntegration(propagate_hub=True), - ], - # This is the recommended format - release= f"oe-plugin@{versionString}", - dist= distType, - environment= "dev" if isDevMode else "production", - before_send= Sentry._beforeSendFilter, - # This means we will send 100% of errors, maybe we want to reduce this in the future? - enable_tracing= enableProfiling, - sample_rate= 1.0, - # Only enable these if we enable profiling. We can't do it in OctoPrint, because it picks up a lot of OctoPrint functions. - traces_sample_rate= 0.01 if enableProfiling else 0.0, - profiles_sample_rate= 0.01 if enableProfiling else 0.0, - ) - except Exception as e: - if Sentry._Logger is not None: - Sentry._Logger.error("Failed to init Sentry: "+str(e)) - - # Set that sentry is ready to use. - Sentry.IsSentrySetup = True + # Disable sentry for now since it seems to be causing a lot of spawned threads and we don't use it right now. + # if Sentry.IsDevMode is False: + # try: + # # We don't want sentry to capture error logs, which is it's default. + # # We do want the logging for breadcrumbs, so we will leave it enabled. + # sentry_logging = LoggingIntegration( + # level=logging.INFO, # Capture info and above as breadcrumbs + # event_level=logging.FATAL # Only send FATAL errors and above. + # ) + # # Setup and init + # sentry_sdk.init( + # dsn= "https://883879bfa2402df86c098f6527f96bfa@oe-sentry.octoeverywhere.com/4", + # integrations= [ + # sentry_logging, + # ThreadingIntegration(propagate_hub=True), + # ], + # # This is the recommended format + # release= f"oe-plugin@{versionString}", + # dist= distType, + # environment= "dev" if isDevMode else "production", + # before_send= Sentry._beforeSendFilter, + # # This means we will send 100% of errors, maybe we want to reduce this in the future? + # enable_tracing= enableProfiling, + # sample_rate= 1.0, + # # Only enable these if we enable profiling. We can't do it in OctoPrint, because it picks up a lot of OctoPrint functions. + # traces_sample_rate= 0.01 if enableProfiling else 0.0, + # profiles_sample_rate= 0.01 if enableProfiling else 0.0, + # ) + # except Exception as e: + # if Sentry._Logger is not None: + # Sentry._Logger.error("Failed to init Sentry: "+str(e)) + + # # Set that sentry is ready to use. + # Sentry.IsSentrySetup = True @staticmethod def SetPrinterId(printerId:str): - sentry_sdk.set_context("octoeverywhere", { "printer-id": printerId }) + #sentry_sdk.set_context("octoeverywhere", { "printer-id": printerId }) + pass @staticmethod @@ -136,7 +138,8 @@ def _beforeSendFilter(event, hint): # Adds a breadcrumb to the sentry log, which is helpful to figure out what happened before an exception. @staticmethod def Breadcrumb(msg:str, data:dict = None, level:str = "info", category:str = "breadcrumb"): - sentry_sdk.add_breadcrumb(message=msg, data=data, level=level, category=category) + #sentry_sdk.add_breadcrumb(message=msg, data=data, level=level, category=category) + pass # Sends an error log to sentry. @@ -147,13 +150,13 @@ def LogError(msg:str, extras:dict = None) -> None: return Sentry._Logger.error(f"Sentry Error: {msg}") # Never send in dev mode, as Sentry will not be setup. - if Sentry.IsSentrySetup and Sentry.IsDevMode is False: - with sentry_sdk.push_scope() as scope: - scope.set_level("error") - if extras is not None: - for key, value in extras.items(): - scope.set_extra(key, value) - sentry_sdk.capture_message(msg) + # if Sentry.IsSentrySetup and Sentry.IsDevMode is False: + # with sentry_sdk.push_scope() as scope: + # scope.set_level("error") + # if extras is not None: + # for key, value in extras.items(): + # scope.set_extra(key, value) + # sentry_sdk.capture_message(msg) # Logs and reports an exception. @@ -194,13 +197,13 @@ def _handleException(msg:str, exception:Exception, sendException:bool, extras:di return # Never send in dev mode, as Sentry will not be setup. - if Sentry.IsSentrySetup and sendException and Sentry.IsDevMode is False: - with sentry_sdk.push_scope() as scope: - scope.set_extra("Exception Message", msg) - if extras is not None: - for key, value in extras.items(): - scope.set_extra(key, value) - sentry_sdk.capture_exception(exception) + # if Sentry.IsSentrySetup and sendException and Sentry.IsDevMode is False: + # with sentry_sdk.push_scope() as scope: + # scope.set_extra("Exception Message", msg) + # if extras is not None: + # for key, value in extras.items(): + # scope.set_extra(key, value) + # sentry_sdk.capture_exception(exception) # If the exception is that we can't start new thread, this logs it, and then restarts if needed. @@ -230,12 +233,12 @@ def _HandleCantCreateThreadException(logger:logging.Logger, e:Exception) -> bool # Flush Sentry # Once this is called, Sentry is shutdown, so we must restart. - try: - client = Hub.current.client - if client is not None: - client.close(timeout=5.0) - except Exception: - pass + # try: + # client = Hub.current.client + # if client is not None: + # client.close(timeout=5.0) + # except Exception: + # pass # Restart the process - We must use this function to actually force the process to exit # The systemd handler will restart us. diff --git a/py_installer/ConfigHelper.py b/py_installer/ConfigHelper.py index f0bfd9f..d0fe85c 100644 --- a/py_installer/ConfigHelper.py +++ b/py_installer/ConfigHelper.py @@ -120,6 +120,8 @@ def WriteBambuDetails(context:Context, accessToken:str, printerSn:str): # Write the new values c.SetStr(Config.SectionBambu, Config.BambuAccessToken, accessToken) c.SetStr(Config.SectionBambu, Config.BambuPrinterSn, printerSn) + # The installer can only setup local connections right now, which is preferred since cloud doesn't work well. + c.SetStr(Config.SectionBambu, Config.BambuConnectionMode, Config.BambuConnectionModeDefault) except Exception as e: Logger.Error("Failed to write bambu details to config. "+str(e)) raise Exception("Failed to write bambu details to config") from e From 1d2278116173724c6b360537d6b2efb3ef5cb719 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Thu, 19 Dec 2024 17:30:19 -0800 Subject: [PATCH 166/328] Version bump! --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 697c68f..6020e52 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this single version string is used by all of the plugins in OctoEverywhere! -plugin_version = "3.6.4" +plugin_version = "3.6.5" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 9a1b50a890b4a7bb6b4fa75e8ac8d6cd1767a4c4 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Mon, 6 Jan 2025 07:33:59 -0800 Subject: [PATCH 167/328] Pulling K1 fix from the OctoApp dev https://github.com/crysxd/OctoApp-Plugin/pull/100 --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 03ccf8a..51e249b 100755 --- a/install.sh +++ b/install.sh @@ -205,7 +205,7 @@ ensure_py_venv() # If not, we will use the version of python built into the system for the existing Creality stuff. if [[ -f /opt/bin/python3 ]] then - virtualenv -p /opt/bin/python3 --system-site-packages "${OE_ENV}" + /opt/bin/virtualenv -p /opt/bin/python3 --system-site-packages "${OE_ENV}" else python3 /usr/lib/python3.8/site-packages/virtualenv.py -p /usr/bin/python3 --system-site-packages "${OE_ENV}" fi From dc60fed2e2b109b86f00b58eef6b3a6b0ddcaeb5 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sun, 19 Jan 2025 12:48:29 -0800 Subject: [PATCH 168/328] Updating the socket retry min and max times. --- octoeverywhere/octoservercon.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/octoeverywhere/octoservercon.py b/octoeverywhere/octoservercon.py index c5ca544..e3782fc 100644 --- a/octoeverywhere/octoservercon.py +++ b/octoeverywhere/octoservercon.py @@ -46,9 +46,12 @@ class OctoServerCon: # we will reconnect quickly again. Remember we always add the random reconnect time as well. WsConnectBackOffSec_Default = 1 # We always add a random second count to the reconnect sleep to add variance. This is the min value. - WsConnectRandomMinSec = 2 + # Remember the server takes about 5 seconds to reboot, so connecting before then is useless. + WsConnectRandomMinSec = 10 # We always add a random second count to the reconnect sleep to add variance. This is the max value. - WsConnectRandomMaxSec = 10 + # Having a wider window allows the client to reconnect at different times, which is good for the server. + WsConnectRandomMaxSec = 30 + def __init__(self, host, endpoint, isPrimaryConnection, shouldUseLowestLatencyServer, printerId, privateKey, logger, uiPopupInvoker, statusChangeHandler, pluginVersion, runForSeconds, summonMethod, serverHostType, isCompanion): self.ProtocolVersion = 1 From 8f827a50757a2f4ae745e67b3b360e90027aaadc Mon Sep 17 00:00:00 2001 From: Yeradon Date: Fri, 24 Jan 2025 12:31:13 +0100 Subject: [PATCH 169/328] sec: remove log of all env variables (#91) --- docker_octoeverywhere/__main__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/docker_octoeverywhere/__main__.py b/docker_octoeverywhere/__main__.py index e50a943..5951619 100644 --- a/docker_octoeverywhere/__main__.py +++ b/docker_octoeverywhere/__main__.py @@ -59,7 +59,6 @@ def CreateDirIfNotExists(path: str) -> None: try: # First, read the required env vars that are set in the dockerfile. - logger.info(f"Env Vars: {os.environ}") virtualEnvPath = EnsureIsPath(os.environ.get("VENV_DIR", None)) repoRootPath = EnsureIsPath(os.environ.get("REPO_DIR", None)) dataPath = EnsureIsPath(os.environ.get("DATA_DIR", None)) From d083535be67eb9888eee5b1c37f9f26be079f790 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Fri, 24 Jan 2025 04:16:20 -0800 Subject: [PATCH 170/328] Updating the bambu reconnect logic to be less aggressive when trying to reconnect to the printer. --- bambu_octoeverywhere/bambuclient.py | 24 ++++++++++++++++-------- linux_host/networksearch.py | 14 +++++++++++--- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/bambu_octoeverywhere/bambuclient.py b/bambu_octoeverywhere/bambuclient.py index bb896d5..3a1851e 100644 --- a/bambu_octoeverywhere/bambuclient.py +++ b/bambu_octoeverywhere/bambuclient.py @@ -155,23 +155,26 @@ def _ClientWorker(self): except Exception as e: if isinstance(e, ConnectionRefusedError): # This means there was no open socket at the given IP and port. - self.Logger.error(f"Failed to connect to the Bambu printer {ipOrHostname}:{self.PortStr}, we will retry in a bit. "+str(e)) + # This happens when the printer is offline, so we only need to log sometimes. + self.Logger.warning(f"Failed to connect to the Bambu printer {ipOrHostname}:{self.PortStr}, we will retry in a bit. "+str(e)) elif isinstance(e, TimeoutError): # This means there was no open socket at the given IP and port. - self.Logger.error(f"Failed to connect to the Bambu printer {ipOrHostname}:{self.PortStr}, we will retry in a bit. "+str(e)) + self.Logger.warning(f"Failed to connect to the Bambu printer {ipOrHostname}:{self.PortStr}, we will retry in a bit. "+str(e)) elif isinstance(e, OSError) and ("Network is unreachable" in str(e) or "No route to host" in str(e)): # This means the IP doesn't route to a device. - self.Logger.error(f"Failed to connect to the Bambu printer {ipOrHostname}:{self.PortStr}, we will retry in a bit. "+str(e)) + self.Logger.warning(f"Failed to connect to the Bambu printer {ipOrHostname}:{self.PortStr}, we will retry in a bit. "+str(e)) elif isinstance(e, socket.timeout) and "timed out" in str(e): # This means the IP doesn't route to a device. - self.Logger.error(f"Failed to connect to the Bambu printer {ipOrHostname}:{self.PortStr} due to a timeout, we will retry in a bit. "+str(e)) + self.Logger.warning(f"Failed to connect to the Bambu printer {ipOrHostname}:{self.PortStr} due to a timeout, we will retry in a bit. "+str(e)) else: # Random other errors. Sentry.Exception(f"Failed to connect to the Bambu printer {ipOrHostname}:{self.PortStr}. We will retry in a bit.", e) # Sleep for a bit between tries. # The main consideration here is to not log too much when the printer is off. But we do still want to connect quickly, when it's back on. - localBackoffCounter = min(localBackoffCounter, 5) + # Note that the system might also do a printer scan after many failed attempts, which can be CPU intensive. + # Right now we allow it to ramp up to 30 seconds between retries. + localBackoffCounter = min(localBackoffCounter, 6) time.sleep(5 * localBackoffCounter) @@ -376,9 +379,11 @@ def _Publish(self, msg:dict) -> bool: def _GetConnectionContextToTry(self) -> ConnectionContext: # Increment and reset if it's too high. # This will restart the process of trying cloud connect and falling back. + doPrinterSearch = False self.ConsecutivelyFailedConnectionAttempts += 1 if self.ConsecutivelyFailedConnectionAttempts > 6: self.ConsecutivelyFailedConnectionAttempts = 0 + doPrinterSearch = True # Get the connection mode set by the user. This defaults to local, but the user can explicitly set it to either. connectionMode = self.Config.GetStr(Config.SectionBambu, Config.BambuConnectionMode, Config.BambuConnectionModeDefault) @@ -392,18 +397,21 @@ def _GetConnectionContextToTry(self) -> ConnectionContext: self.Logger.warning("We tried to connect via Bambu Cloud, but failed. We will try a local connection.") # On the first few attempts, use the expected IP or the cloud config. - # The first attempt will always be attempt 1, since it's reset to 0 and incremented before connecting. + # Every time we reset the count, we will try a network scan to see if we can find the printer guessing it's IP might have changed. # The IP can be empty, like if the docker container is used, in which case we should always search for the printer. configIpOrHostname = self.Config.GetStr(Config.SectionCompanion, Config.CompanionKeyIpOrHostname, None) - if self.ConsecutivelyFailedConnectionAttempts < 4: + if doPrinterSearch is False: # If we aren't using a cloud connection or it failed, return the local hostname if configIpOrHostname is not None and len(configIpOrHostname) > 0: return self._GetLocalConnectionContext(configIpOrHostname) # If we fail too many times, try to scan for the printer on the local subnet, the IP could have changed. # Since we 100% identify the printer by the access token and printer SN, we can try to scan for it. + # Note we don't want to do this too often since it's CPU intensive and the printer might just be off. + # We use a lower thread count and delay before each action to reduce the required load. + # Using this config, it takes about 30 seconds to scan for the printer. self.Logger.info(f"Searching for your Bambu Lab printer {self.PrinterSn}") - ips = NetworkSearch.ScanForInstances_Bambu(self.Logger, self.LanAccessCode, self.PrinterSn) + ips = NetworkSearch.ScanForInstances_Bambu(self.Logger, self.LanAccessCode, self.PrinterSn, threadCount=25, delaySec=0.2) # If we get an IP back, it is the printer. # The scan above will only return an IP if the printer was successfully connected to, logged into, and fully authorized with the Access Token and Printer SN. diff --git a/linux_host/networksearch.py b/linux_host/networksearch.py index 55a6ed5..0d3daac 100644 --- a/linux_host/networksearch.py +++ b/linux_host/networksearch.py @@ -1,4 +1,5 @@ import ssl +import time import json import socket import logging @@ -32,12 +33,17 @@ class NetworkSearch: # Scans the local IP LAN subset for Bambu servers that successfully authorize given the access code and printer sn. + # Thread count and delay can be used to control how aggressive the scan is. @staticmethod - def ScanForInstances_Bambu(logger:logging.Logger, accessCode:str, printerSn:str, portStr:str = None) -> List[str]: + def ScanForInstances_Bambu(logger:logging.Logger, accessCode:str, printerSn:str, portStr:str = None, threadCount:int=None, delaySec:float=0.0) -> List[str]: def callback(ip:str): + # This is a quick fix to slow down the scan so it doesn't eat a lot of CPU load on the device while the printer is off + # and the plugin is trying to find it. But it's important this scan also be fast, for the installer. + if delaySec > 0: + time.sleep(delaySec) return NetworkSearch.ValidateConnection_Bambu(logger, ip, accessCode, printerSn, portStr, timeoutSec=5) # We want to return if any one IP is found, since there can only be one printer that will match the printer 100% correct. - return NetworkSearch._ScanForInstances(logger, callback, returnAfterNumberFound=1) + return NetworkSearch._ScanForInstances(logger, callback, returnAfterNumberFound=1, threadCount=threadCount) # The final two steps can happen in different orders, so we need to wait for both the sub success and state object to be received. @@ -193,7 +199,7 @@ def message(client, userdata:dict, mqttMsg:mqtt.MQTTMessage): # testConFunction must be a function func(ip:str) -> NetworkValidationResult # Returns a list of IPs that reported Success() == True @staticmethod - def _ScanForInstances(logger:logging.Logger, testConFunction, returnAfterNumberFound = 0) -> List[str]: + def _ScanForInstances(logger:logging.Logger, testConFunction, returnAfterNumberFound:int = 0, threadCount:int = None) -> List[str]: foundIps = [] try: localIp = NetworkSearch._TryToGetLocalIp() @@ -215,6 +221,8 @@ def _ScanForInstances(logger:logging.Logger, testConFunction, returnAfterNumberF # if an exception was thrown in the thread, it would hang the system. # I fixed that but also lowered the concurrent thread count to 100, which seems more comfortable. totalThreads = 100 + if threadCount is not None: + totalThreads = threadCount outstandingIpsToCheck = [] counter = 0 while counter < 255: From 6542fc2a74693dbdd119be81e01ecdf65363d1cc Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Fri, 24 Jan 2025 04:25:03 -0800 Subject: [PATCH 171/328] Github actions fix --- .github/workflows/pylint.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 187faf9..5f9008b 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -10,7 +10,8 @@ jobs: strategy: matrix: # The sonic pad runs 3.7, so it's important to keep it here to make sure all of our required dependencies work - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + # Github removed support for 3.7, so we need to find another way to test it. + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} From 7c73ba9ffa04ec951ad5154bc943440042112b4f Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Fri, 24 Jan 2025 04:40:08 -0800 Subject: [PATCH 172/328] Actions change. --- .github/workflows/pylint.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 5f9008b..68134fb 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -6,12 +6,12 @@ on: jobs: build: - runs-on: ubuntu-latest + # We must limit the OS to ubuntu-22.04 instead of the default ubuntu-latest, to keep PY3.7 available. + runs-on: ubuntu-22.04 strategy: matrix: # The sonic pad runs 3.7, so it's important to keep it here to make sure all of our required dependencies work - # Github removed support for 3.7, so we need to find another way to test it. - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} From 92d9d1f528c924157cfd509296005c91778598e7 Mon Sep 17 00:00:00 2001 From: jwilson2899 <69737157+jwilson2899@users.noreply.github.com> Date: Sun, 26 Jan 2025 13:13:53 -0500 Subject: [PATCH 173/328] Added tzdata to allow time zone to be set properly. (#93) * Update Dockerfile Added entries to install TZ data package * Added value for timezone Added entry to set timezone to proper value * Correct typo Corrected type in sample TZ entry. --- Dockerfile | 7 +++++++ docker-compose.yml | 5 ++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 806ffac..421f4ed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,6 +22,13 @@ ENV DATA_DIR=/data/ # GCC, python3-dev, and musl-dev are required for pillow, and jpeg-dev and zlib-dev are required for jpeg support. RUN apk add --no-cache curl ffmpeg jq python3 python3-dev gcc musl-dev py3-pip py3-virtualenv jpeg-dev libjpeg-turbo-dev zlib-dev py3-pillow libffi-dev +# Timezone setup steps +# These steps are necessary to add timezone support to the container and allow for setting the timezone +# This allows the log files to show the correct local time +RUN apk add --no-cache tzdata +ENV TZ=Etc/GMT +RUN cp /usr/share/zoneinfo/Etc/GMT /etc/localtime + # # We decided to not run the installer, since the point of the installer is to setup the env, build the launch args, and setup the service. # Instead, we will manually run the smaller subset of commands that are required to get the env setup in docker. diff --git a/docker-compose.yml b/docker-compose.yml index be4eac8..765d560 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,9 @@ services: - SERIAL_NUMBER=XXXXXXXXXXXXXXX # Find using the printer's display or use https://octoeverywhere.com/s/bambu-ip - PRINTER_IP=XXX.XXX.XXX.XXX + # Set timezone to proper timezone for logs using standard timezones: + # https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List + - TZ=America/New_York # Optionally: If you want to connect via the Bambu Cloud, you can specify the following environment variables. # By default the plugin will use the local connection mode, which is preferred. @@ -47,4 +50,4 @@ services: # # volumes: # # Specify a path mapping for the required persistent storage - # - ./data:/data \ No newline at end of file + # - ./data:/data From 6258909391cb789a86c2cf8a2971fcce8608495b Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Sun, 26 Jan 2025 10:19:30 -0800 Subject: [PATCH 174/328] Version bump --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6020e52..2560c0b 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this single version string is used by all of the plugins in OctoEverywhere! -plugin_version = "3.6.5" +plugin_version = "3.6.6" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 24759466cbd06dfa0169aa8a9a2e2f797c50531e Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Tue, 28 Jan 2025 09:39:44 -0800 Subject: [PATCH 175/328] Removing the Sentry SDK depdency to fix the new OctoPrint RC. --- requirements.txt | 4 ++-- setup.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 49f2c2d..a884b19 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ # OctoPrint uses the package list in the setup.py file. # # BUT, for the most part, the packages should be exactly synced between these sources. -# The only excpetion would be any packages moonraker or OctoPrint depend upon, that the other doesn't. +# The only exception would be any packages moonraker or OctoPrint depend upon, that the other doesn't. # # For comments on package lock versions, see the comments in the setup.py file. # @@ -16,7 +16,7 @@ rsa>=4.9 dnspython>=2.3.0 httpx>=0.24.1,<0.26.0 urllib3>=1.26.15,<2.0.0 -sentry-sdk>=1.19.1,<2 +#sentry-sdk>=1.19.1,<2 #zstandard>=0.22.0,<0.23.0 # The following are required only for Moonraker diff --git a/setup.py b/setup.py index 2560c0b..80b93ec 100644 --- a/setup.py +++ b/setup.py @@ -73,7 +73,7 @@ # certifi - We use to keep certs on the device that we need for let's encrypt. So we want to keep it fresh. # rsa - OctoPrint 1.5.3 requires RAS>=4.0, so we must leave it at 4.0. # httpx - Is an asyncio http lib. It seems to be required by dnspython, but dnspython doesn't enforce it. We had a user having an issue that updated to 0.24.0, and it resolved the issue. -# sentry-sdk - We use the same version as OctoPrint, so we don't have to worry about mismatched versions. +# sentry-sdk - We don't use Sentry right now, so we disabled it. It was conflicting with the new OctoPrint RC, so if we add it back, we need to address that. # # Note! These also need to stay in sync with requirements.txt, for the most part they should be the exact same! plugin_requires = [ @@ -86,7 +86,7 @@ "dnspython>=2.3.0", "httpx>=0.24.1,<0.26.0", "urllib3>=1.26.18,<2.0.0", - "sentry-sdk>=1.19.1,<2", + #"sentry-sdk>=TODO", #"zstandard" - optional lib see notes ] From eedce4a3a2768ff7e994ca719efd448762342b36 Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Tue, 28 Jan 2025 09:43:12 -0800 Subject: [PATCH 176/328] Enabling Auto Escaped Templates & Version bump. --- octoprint_octoeverywhere/__init__.py | 5 +++++ setup.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/octoprint_octoeverywhere/__init__.py b/octoprint_octoeverywhere/__init__.py index 1b08af3..1c5d9c7 100644 --- a/octoprint_octoeverywhere/__init__.py +++ b/octoprint_octoeverywhere/__init__.py @@ -77,6 +77,11 @@ def is_wizard_required(self): def get_wizard_version(self): return 10 + # Turns on auto escaping for the template. + # Improves security, recommended here: https://community.octoprint.org/t/how-do-i-improve-my-plugins-security-by-enabling-autoescape/61067 + def is_template_autoescaped(self): + return True + def get_wizard_details(self): # Do some sanity checking logic, since this has been sensitive in the past. printerUrl = self.GetAddPrinterUrl() diff --git a/setup.py b/setup.py index 80b93ec..043bb4c 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # The plugin's version. Can be overwritten within OctoPrint's internal data via __plugin_version__ in the plugin module # Note that this single version string is used by all of the plugins in OctoEverywhere! -plugin_version = "3.6.6" +plugin_version = "3.6.7" # The plugin's description. Can be overwritten within OctoPrint's internal data via __plugin_description__ in the plugin # module From 0bee695f5cecbd9546bc2c23d45bf2530492800e Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Tue, 28 Jan 2025 17:13:01 -0800 Subject: [PATCH 177/328] Updating the OctoEverywhere OctoPrint UI pages. --- .../templates/octoeverywhere_settings.jinja2 | 53 ++++++++++--------- .../templates/octoeverywhere_wizard.jinja2 | 29 ++++++++-- setup.py | 2 +- 3 files changed, 54 insertions(+), 30 deletions(-) diff --git a/octoprint_octoeverywhere/templates/octoeverywhere_settings.jinja2 b/octoprint_octoeverywhere/templates/octoeverywhere_settings.jinja2 index 05fef8c..4775db7 100644 --- a/octoprint_octoeverywhere/templates/octoeverywhere_settings.jinja2 +++ b/octoprint_octoeverywhere/templates/octoeverywhere_settings.jinja2 @@ -9,83 +9,88 @@

Blast off 🚀 with OctoEverywhere.com! Access your full OctoPrint portal, printer controls, camera streams, and plugins from anywhere! OctoEverywhere works on your desktop, laptop, tablet, phone, and your favorite OctoPrint apps!

OctoEverywhere is free, secure, and super easy to setup!

- Finish Your Two Minute Setup Now!   + Finish Your 20 Second Setup Now!