From 663720a393372b1c6b3af3023366c901c836fe48 Mon Sep 17 00:00:00 2001 From: jeffy-g Date: Sun, 1 Dec 2024 02:11:19 +0900 Subject: [PATCH 01/12] fix localized import problem Fitting description limit extended to 500 characters localized fitting file sample --- config.py | 8 + dist_assets/win/dist.py | 2 +- eos/db/saveddata/queries.py | 2 + eos/db/util.py | 2 + gui/builtinAdditionPanes/notesView.py | 57 +- gui/builtinItemStatsViews/itemDescription.py | 4 +- gui/globalEvents.py | 32 +- gui/mainFrame.py | 3 +- pyfa.spec | 2 + service/fit.py | 1 + service/market.py | 1 + service/port/port.py | 19 +- service/port/xml.py | 60 +- tests/jeffy_ja-en[99].xml | 2116 ------------------ tests/localized-fitting-files/ch_ch.xml | 33 + tests/localized-fitting-files/ch_en.xml | 33 + tests/localized-fitting-files/de_de.xml | 33 + tests/localized-fitting-files/de_en.xml | 33 + tests/localized-fitting-files/es_en.xml | 33 + tests/localized-fitting-files/es_es.xml | 33 + tests/localized-fitting-files/fr_en.xml | 33 + tests/localized-fitting-files/fr_fr.xml | 33 + tests/localized-fitting-files/ja_en.xml | 33 + tests/localized-fitting-files/ja_ja.xml | 33 + tests/localized-fitting-files/ko_en.xml | 33 + tests/localized-fitting-files/ko_ko.xml | 33 + tests/localized-fitting-files/readme.md | 3 + tests/localized-fitting-files/ru_en.xml | 33 + tests/localized-fitting-files/ru_ru.xml | 33 + tests/test_unread_desc.py | 2 +- utils/strfunctions.py | 6 +- 31 files changed, 597 insertions(+), 2185 deletions(-) delete mode 100644 tests/jeffy_ja-en[99].xml create mode 100644 tests/localized-fitting-files/ch_ch.xml create mode 100644 tests/localized-fitting-files/ch_en.xml create mode 100644 tests/localized-fitting-files/de_de.xml create mode 100644 tests/localized-fitting-files/de_en.xml create mode 100644 tests/localized-fitting-files/es_en.xml create mode 100644 tests/localized-fitting-files/es_es.xml create mode 100644 tests/localized-fitting-files/fr_en.xml create mode 100644 tests/localized-fitting-files/fr_fr.xml create mode 100644 tests/localized-fitting-files/ja_en.xml create mode 100644 tests/localized-fitting-files/ja_ja.xml create mode 100644 tests/localized-fitting-files/ko_en.xml create mode 100644 tests/localized-fitting-files/ko_ko.xml create mode 100644 tests/localized-fitting-files/readme.md create mode 100644 tests/localized-fitting-files/ru_en.xml create mode 100644 tests/localized-fitting-files/ru_ru.xml diff --git a/config.py b/config.py index ddfd1324f1..784432947c 100644 --- a/config.py +++ b/config.py @@ -66,6 +66,14 @@ CATALOG = 'lang' +EVE_FIT_NOTE_MAX = 500 +''' +eve fit (xml) "description" limit + +Description can contain html tags like + +If it contains html tags, they will be converted to html entities +''' slotColourMapDark = { FittingSlot.LOW: wx.Colour(44, 36, 19), # yellow = low slots 24/13 diff --git a/dist_assets/win/dist.py b/dist_assets/win/dist.py index c4663bc1d0..be543ee201 100644 --- a/dist_assets/win/dist.py +++ b/dist_assets/win/dist.py @@ -14,7 +14,7 @@ os.environ["PYFA_DIST_DIR"] = os.path.join(os.getcwd(), 'dist') os.environ["PYFA_VERSION"] = version -iscc = r"C:\Program Files (x86)\Inno Setup 6\ISCC.exe" +iscc = "C:\Program Files (x86)\Inno Setup 6\ISCC.exe" source = os.path.join(os.environ["PYFA_DIST_DIR"], "pyfa") diff --git a/eos/db/saveddata/queries.py b/eos/db/saveddata/queries.py index 4d4ad8f5c5..42d5263d11 100644 --- a/eos/db/saveddata/queries.py +++ b/eos/db/saveddata/queries.py @@ -205,6 +205,7 @@ def getCharactersForUser(lookfor, eager=None): @cachedQuery(Fit, 1, "lookfor") def getFit(lookfor, eager=None): + # type: (int, bool) -> Fit if isinstance(lookfor, int): if eager is None: with sd_lock: @@ -319,6 +320,7 @@ def countFitsWithShip(lookfor, ownerID=None, where=None, eager=None): def getFitList(eager=None): + # type: (list[str]) -> list[Fit] eager = processEager(eager) with sd_lock: fits = removeInvalid(saveddata_session.query(Fit).options(*eager).all()) diff --git a/eos/db/util.py b/eos/db/util.py index 7fcf3504d0..47a7bbb66d 100644 --- a/eos/db/util.py +++ b/eos/db/util.py @@ -35,6 +35,7 @@ def processEager(eager): + # type: (list[str]) -> str if eager is None: return tuple() else: @@ -49,6 +50,7 @@ def processEager(eager): def _replacements(eagerString): + # type: (str) -> str splitEager = eagerString.split(".") for i in range(len(splitEager)): part = splitEager[i] diff --git a/gui/builtinAdditionPanes/notesView.py b/gui/builtinAdditionPanes/notesView.py index 44c49d9739..aca6fbd438 100644 --- a/gui/builtinAdditionPanes/notesView.py +++ b/gui/builtinAdditionPanes/notesView.py @@ -6,6 +6,30 @@ from gui.utils.helpers_wxPython import HandleCtrlBackspace from gui.utils.numberFormatter import formatAmount from service.fit import Fit +from config import EVE_FIT_NOTE_MAX + + +LATER = 1000 +'''timer interval, delay the save''' + +# 3 +EXPAND_LF_LEN = len("
") - 1 +''' +If you save `Fit.notes` to "description" in eve fit(xml export), +newline characters must be converted to "
" +''' + +def computeEVEFitDescSize(note): + # type: (str) -> int + return len(note) + (note.count("\n") * EXPAND_LF_LEN) + +def ifExceedsTheUpperLimit(nv, note=None): + # type: (wx.TextCtrl, str) -> None + '''When the note size exceeds the upper limit, the text will turn red.''' + if note is None: note = nv.GetValue() + color = '#FF0000' if computeEVEFitDescSize(note) > EVE_FIT_NOTE_MAX else '#000000' + nv.SetForegroundColour(color) + nv.Refresh(False) class NotesView(wx.Panel): @@ -13,21 +37,23 @@ class NotesView(wx.Panel): def __init__(self, parent): wx.Panel.__init__(self, parent) self.lastFitId = None + self.changeTimer = wx.Timer(self) self.mainFrame = gui.mainFrame.MainFrame.getInstance() - mainSizer = wx.BoxSizer(wx.VERTICAL) self.editNotes = wx.TextCtrl(self, style=wx.TE_MULTILINE | wx.BORDER_NONE) - mainSizer.Add(self.editNotes, 1, wx.EXPAND | wx.ALL, 10) - self.SetSizer(mainSizer) - self.mainFrame.Bind(GE.FIT_CHANGED, self.fitChanged) self.Bind(wx.EVT_TEXT, self.onText) - self.editNotes.Bind(wx.EVT_KEY_DOWN, self.OnKeyDown) - self.changeTimer = wx.Timer(self) self.Bind(wx.EVT_TIMER, self.delayedSave, self.changeTimer) + self.mainFrame.Bind(GE.FIT_CHANGED, self.fitChanged) + self.editNotes.Bind(wx.EVT_KEY_DOWN, self.OnKeyDown) + mainSizer = wx.BoxSizer(wx.VERTICAL) + mainSizer.Add(self.editNotes, 1, wx.EXPAND | wx.ALL, 10) + self.SetSizer(mainSizer) def OnKeyDown(self, event): + # type: (wx.KeyEvent) -> None + nv = self.editNotes if event.RawControlDown() and event.GetKeyCode() == wx.WXK_BACK: try: - HandleCtrlBackspace(self.editNotes) + HandleCtrlBackspace(nv) except (KeyboardInterrupt, SystemExit): raise except: @@ -35,7 +61,10 @@ def OnKeyDown(self, event): else: event.Skip() + ifExceedsTheUpperLimit(nv) + def fitChanged(self, event): + # type: (wx.Event) -> None event.Skip() activeFitID = self.mainFrame.getActiveFit() if activeFitID is not None and activeFitID not in event.fitIDs: @@ -57,21 +86,29 @@ def fitChanged(self, event): return elif activeFitID != self.lastFitId: self.lastFitId = activeFitID - self.editNotes.ChangeValue(fit.notes or "") + note = fit.notes or "" + nv = self.editNotes + nv.ChangeValue(note) + ifExceedsTheUpperLimit(nv, note) wx.PostEvent(self.mainFrame, GE.FitNotesChanged()) def onText(self, event): + # type: (wx.Event) -> None # delay the save so we're not writing to sqlite on every keystroke self.changeTimer.Stop() # cancel the existing timer - self.changeTimer.Start(1000, True) + self.changeTimer.Start(LATER, True) + # When the note size exceeds the upper limit, the text will turn red. + ifExceedsTheUpperLimit(self.editNotes) def delayedSave(self, event): + # type: (wx.Event) -> None event.Skip() sFit = Fit.getInstance() sFit.editNotes(self.lastFitId, self.editNotes.GetValue()) wx.PostEvent(self.mainFrame, GE.FitNotesChanged()) def getTabExtraText(self): + # type: () -> str|None fitID = self.mainFrame.getActiveFit() if fitID is None: return None @@ -82,7 +119,7 @@ def getTabExtraText(self): opt = sFit.serviceFittingOptions["additionsLabels"] # Amount of active implants if opt in (1, 2): - amount = len(self.editNotes.GetValue()) + amount = computeEVEFitDescSize(self.editNotes.GetValue()) return ' ({})'.format(formatAmount(amount, 2, 0, 3)) if amount else None else: return None diff --git a/gui/builtinItemStatsViews/itemDescription.py b/gui/builtinItemStatsViews/itemDescription.py index 50292e361a..6f9cf550dd 100644 --- a/gui/builtinItemStatsViews/itemDescription.py +++ b/gui/builtinItemStatsViews/itemDescription.py @@ -22,9 +22,9 @@ def __init__(self, parent, stuff, item): desc = item.description.replace("\n", "
") # Strip font tags - desc = re.sub("<( *)font( *)color( *)=(.*?)>(?P.*?)<( *)/( *)font( *)>", r"\g", desc) + desc = re.sub(r"<( *)font( *)color( *)=(.*?)>(?P.*?)<( *)/( *)font( *)>", r"\g", desc) # Strip URLs - desc = re.sub("<( *)a(.*?)>(?P.*?)<( *)/( *)a( *)>", r"\g", desc) + desc = re.sub(r"<( *)a(.*?)>(?P.*?)<( *)/( *)a( *)>", r"\g", desc) desc = "{}".format( bgcolor.GetAsString(wx.C2S_HTML_SYNTAX), fgcolor.GetAsString(wx.C2S_HTML_SYNTAX), diff --git a/gui/globalEvents.py b/gui/globalEvents.py index 0f74f663ee..1d51de3b2b 100644 --- a/gui/globalEvents.py +++ b/gui/globalEvents.py @@ -1,22 +1,22 @@ # noinspection PyPackageRequirements -import wx.lib.newevent +from wx.lib.newevent import NewEvent -FitRenamed, FIT_RENAMED = wx.lib.newevent.NewEvent() -FitChanged, FIT_CHANGED = wx.lib.newevent.NewEvent() -FitRemoved, FIT_REMOVED = wx.lib.newevent.NewEvent() -FitNotesChanged, FIT_NOTES_CHANGED = wx.lib.newevent.NewEvent() -CharListUpdated, CHAR_LIST_UPDATED = wx.lib.newevent.NewEvent() -CharChanged, CHAR_CHANGED = wx.lib.newevent.NewEvent() -GraphOptionChanged, GRAPH_OPTION_CHANGED = wx.lib.newevent.NewEvent() -TargetProfileRenamed, TARGET_PROFILE_RENAMED = wx.lib.newevent.NewEvent() -TargetProfileChanged, TARGET_PROFILE_CHANGED = wx.lib.newevent.NewEvent() -TargetProfileRemoved, TARGET_PROFILE_REMOVED = wx.lib.newevent.NewEvent() +FitRenamed, FIT_RENAMED = NewEvent() +FitChanged, FIT_CHANGED = NewEvent() +FitRemoved, FIT_REMOVED = NewEvent() +FitNotesChanged, FIT_NOTES_CHANGED = NewEvent() +CharListUpdated, CHAR_LIST_UPDATED = NewEvent() +CharChanged, CHAR_CHANGED = NewEvent() +GraphOptionChanged, GRAPH_OPTION_CHANGED = NewEvent() +TargetProfileRenamed, TARGET_PROFILE_RENAMED = NewEvent() +TargetProfileChanged, TARGET_PROFILE_CHANGED = NewEvent() +TargetProfileRemoved, TARGET_PROFILE_REMOVED = NewEvent() # For events when item is actually replaced under the hood, # but from user's perspective it's supposed to change/mutate -ItemChangedInplace, ITEM_CHANGED_INPLACE = wx.lib.newevent.NewEvent() +ItemChangedInplace, ITEM_CHANGED_INPLACE = NewEvent() -EffectiveHpToggled, EFFECTIVE_HP_TOGGLED = wx.lib.newevent.NewEvent() +EffectiveHpToggled, EFFECTIVE_HP_TOGGLED = NewEvent() -SsoLoggingIn, EVT_SSO_LOGGING_IN = wx.lib.newevent.NewEvent() -SsoLogin, EVT_SSO_LOGIN = wx.lib.newevent.NewEvent() -SsoLogout, EVT_SSO_LOGOUT = wx.lib.newevent.NewEvent() +SsoLoggingIn, EVT_SSO_LOGGING_IN = NewEvent() +SsoLogin, EVT_SSO_LOGIN = NewEvent() +SsoLogout, EVT_SSO_LOGOUT = NewEvent() diff --git a/gui/mainFrame.py b/gui/mainFrame.py index 44bfc5c496..169d31b4a4 100644 --- a/gui/mainFrame.py +++ b/gui/mainFrame.py @@ -363,6 +363,7 @@ def UnregisterStatsWindow(self, wnd): self.statsWnds.remove(wnd) def getActiveFit(self): + # type: () -> int p = self.fitMultiSwitch.GetSelectedPage() m = getattr(p, "getActiveFit", None) return m() if m is not None else None @@ -845,7 +846,7 @@ def fileImportDialog(self, event): style=wx.FD_OPEN | wx.FD_FILE_MUST_EXIST | wx.FD_MULTIPLE ) as dlg: if dlg.ShowModal() == wx.ID_OK: - # set some arbitrary spacing to create width in window + # set some arbitrary spacing to create width in window progress = ProgressHelper(message=" " * 100, callback=self._openAfterImport) call = (Port.importFitsThreaded, [dlg.GetPaths(), progress], {}) self.handleProgress( diff --git a/pyfa.spec b/pyfa.spec index 33fb64e0ad..28a219b6da 100644 --- a/pyfa.spec +++ b/pyfa.spec @@ -1,5 +1,7 @@ # -*- mode: python -*- +# If you get a cp65001 error during build, set the environment variable PYTHONIOENCODING="utf8" + import os from itertools import chain import subprocess diff --git a/service/fit.py b/service/fit.py index 4e3a1aa7bc..6c85ba3cf1 100644 --- a/service/fit.py +++ b/service/fit.py @@ -321,6 +321,7 @@ def switchFit(self, fitID): self.fill(fit) def getFit(self, fitID, projected=False, basic=False): + # type: (int, bool, bool) -> Fit """ Gets fit from database diff --git a/service/market.py b/service/market.py index 9db615e2fb..4659f15ef5 100644 --- a/service/market.py +++ b/service/market.py @@ -495,6 +495,7 @@ def __makeReverseMetaMapIndices(self): @staticmethod def getItem(identity, *args, **kwargs): + # type: (str|int|types_Item, list[str], dict[str]) -> types_Item|None """Get item by its ID or name""" try: if isinstance(identity, types_Item): diff --git a/service/port/port.py b/service/port/port.py index cb10994715..fde33b06ab 100644 --- a/service/port/port.py +++ b/service/port/port.py @@ -73,9 +73,11 @@ def is_tag_replace(cls): @staticmethod def backupFits(path, progress): + # type: (str, object) -> None pyfalog.debug("Starting backup fits thread.") def backupFitsWorkerFunc(path, progress): + # type: (str, object) -> None try: backedUpFits = Port.exportXml(svcFit.getInstance().getAllFits(), progress) if backedUpFits: @@ -98,17 +100,14 @@ def backupFitsWorkerFunc(path, progress): @staticmethod def importFitsThreaded(paths, progress): + # type: (list[str], object) -> None """ :param paths: fits data file path list. :rtype: None """ pyfalog.debug("Starting import fits thread.") - - def importFitsFromFileWorkerFunc(paths, progress): - Port.importFitFromFiles(paths, progress) - threading.Thread( - target=importFitsFromFileWorkerFunc, + target=Port.importFitFromFiles, args=(paths, progress) ).start() @@ -127,7 +126,7 @@ def importFitFromFiles(paths, progress=None): try: for path in paths: if progress: - if progress and progress.userCancelled: + if progress.userCancelled: progress.workerWorking = False return False, "Cancelled by user" msg = "Processing file:\n%s" % path @@ -212,6 +211,7 @@ def importFitFromBuffer(bufferStr, activeFit=None): @classmethod def importAuto(cls, string, path=None, activeFit=None, progress=None): + # type: (str, str, svcFit, object) -> None lines = string.splitlines() # Get first line and strip space symbols of it to avoid possible detection errors firstLine = '' @@ -231,21 +231,21 @@ def importAuto(cls, string, path=None, activeFit=None, progress=None): # If we've got source file name which is used to describe ship name # and first line contains something like [setup name], detect as eft config file - if re.match(r"^\s*\[.*\]", firstLine) and path is not None: + if re.match(r"^\s*\[.*]", firstLine) and path is not None: filename = os.path.split(path)[1] shipName = filename.rsplit('.')[0] return "EFT Config", True, cls.importEftCfg(shipName, lines, progress) # If no file is specified and there's comma between brackets, # consider that we have [ship, setup name] and detect like eft export format - if re.match(r"^\s*\[.*,.*\]", firstLine): + if re.match(r"^\s*\[.*,.*]", firstLine): return "EFT", True, (cls.importEft(lines),) # Check if string is in DNA format dnaPattern = r"\d+(:\d+(;\d+))*::" if re.match(dnaPattern, firstLine): return "DNA", True, (cls.importDna(string),) - dnaChatPattern = r"{})>(?P[^<>]+)".format(dnaPattern) + dnaChatPattern = "{})>(?P[^<>]+)".format(dnaPattern) m = re.search(dnaChatPattern, firstLine) if m: return "DNA", True, (cls.importDna(m.group("dna"), fitName=m.group("fitName")),) @@ -332,6 +332,7 @@ def importXml(text, progress=None): @staticmethod def exportXml(fits, progress=None, callback=None): + # type: (list[svcFit], object, object) -> str return exportXml(fits, progress, callback=callback) # Multibuy-related methods diff --git a/service/port/xml.py b/service/port/xml.py index 4fbb161e20..fe17ba84c1 100644 --- a/service/port/xml.py +++ b/service/port/xml.py @@ -18,8 +18,8 @@ # ============================================================================= import re -import xml.dom -import xml.parsers.expat +from xml.dom import minidom +# import xml.parsers.expat from logbook import Logger @@ -38,6 +38,7 @@ from service.port.muta import renderMutantAttrs, parseMutantAttrs from service.port.shared import fetchItem from utils.strfunctions import replace_ltgt, sequential_rep +from config import EVE_FIT_NOTE_MAX pyfalog = Logger(__name__) @@ -46,14 +47,12 @@ RE_LTGT = "&(lt|gt);" L_MARK = "<localized hint="" # <localized hint="([^"]+)">([^\*]+)\*<\/localized> -LOCALIZED_PATTERN = re.compile(r'([^\*]+)\*') - - +LOCALIZED_PATTERN = re.compile(r'([^*]+)\*?') class ExtractingError(Exception): pass - def _extract_match(t): + # type: (str) -> tuple[str, str] m = LOCALIZED_PATTERN.match(t) if m is None: raise ExtractingError @@ -61,14 +60,14 @@ def _extract_match(t): return m.group(1), m.group(2) -def _resolve_ship(fitting, sMkt, b_localized): - # type: (xml.dom.minidom.Element, service.market.Market, bool) -> eos.saveddata.fit.Fit +def _solve_ship(fitting, sMkt, b_localized): + # type: (minidom.Element, Market, bool) -> Fit """ NOTE: Since it is meaningless unless a correct ship object can be constructed, process flow changed """ # ------ Confirm ship # Maelstrom - shipType = fitting.getElementsByTagName("shipType").item(0).getAttribute("value") + shipType = fitting.getElementsByTagName("shipType")[0].getAttribute("value") anything = None if b_localized: try: @@ -89,7 +88,7 @@ def _resolve_ship(fitting, sMkt, b_localized): except (KeyboardInterrupt, SystemExit): raise except Exception as e: - pyfalog.warning("Caught exception on _resolve_ship") + pyfalog.warning("Caught exception on _solve_ship") pyfalog.error(e) limit -= 1 if limit == 0: @@ -115,8 +114,8 @@ def _resolve_ship(fitting, sMkt, b_localized): return fitobj -def _resolve_module(hardware, sMkt, b_localized): - # type: (xml.dom.minidom.Element, service.market.Market, bool) -> eos.saveddata.module.Module +def _solve_module(hardware, sMkt, b_localized): + # type: (minidom.Element, Market, bool) -> Module moduleName = hardware.getAttribute("base_type") or hardware.getAttribute("type") emergency = None if b_localized: @@ -132,19 +131,25 @@ def _resolve_module(hardware, sMkt, b_localized): must_retry = False try: item = sMkt.getItem(moduleName, eager="group.category") + if not item: + raise ValueError(f"{moduleName} is not valid") + pyfalog.info('_solve_module - sMkt.getItem: {}', item) except (KeyboardInterrupt, SystemExit): raise except Exception as e: - pyfalog.warning("Caught exception on _resolve_module") + pyfalog.warning("Caught exception on _solve_module, name:{}", moduleName) pyfalog.error(e) limit -= 1 if limit == 0: break moduleName = emergency must_retry = True - if not must_retry: + if not must_retry and item: break + if item is None: + raise Exception("cannot resolve module or item.") + mutaplasmidName = hardware.getAttribute("mutaplasmid") mutaplasmidItem = fetchItem(mutaplasmidName) if mutaplasmidName else None @@ -155,15 +160,16 @@ def _resolve_module(hardware, sMkt, b_localized): def importXml(text, progress): + # type: (str, object) -> list[Fit] from .port import Port sMkt = Market.getInstance() - doc = xml.dom.minidom.parseString(text) + doc = minidom.parseString(text) + # NOTE: # When L_MARK is included at this point, # Decided to be localized data b_localized = L_MARK in text - fittings = doc.getElementsByTagName("fittings").item(0) - fittings = fittings.getElementsByTagName("fitting") + fittings = doc.getElementsByTagName("fitting") fit_list = [] failed = 0 @@ -172,7 +178,7 @@ def importXml(text, progress): return [] try: - fitobj = _resolve_ship(fitting, sMkt, b_localized) + fitobj = _solve_ship(fitting, sMkt, b_localized) except (KeyboardInterrupt, SystemExit): raise except: @@ -181,7 +187,7 @@ def importXml(text, progress): # -- 170327 Ignored description -- # read description from exported xml. (EVE client, EFT) - description = fitting.getElementsByTagName("description").item(0).getAttribute("value") + description = fitting.getElementsByTagName("description")[0].getAttribute("value") if description is None: description = "" elif len(description): @@ -193,10 +199,12 @@ def importXml(text, progress): fitobj.notes = description hardwares = fitting.getElementsByTagName("hardware") + # Sorting by "slot" attr is cool + hardwares.sort(key=lambda e: e.getAttribute("slot")) moduleList = [] for hardware in hardwares: try: - item, mutaItem, mutaAttrs = _resolve_module(hardware, sMkt, b_localized) + item, mutaItem, mutaAttrs = _solve_module(hardware, sMkt, b_localized) if not item or not item.published: continue @@ -279,9 +287,9 @@ def importXml(text, progress): return fit_list - def exportXml(fits, progress, callback): - doc = xml.dom.minidom.Document() + # type: (list[Fit], object, any) -> str|None + doc = minidom.Document() fittings = doc.createElement("fittings") # fit count fit_count = len(fits) @@ -310,11 +318,11 @@ def addMutantAttributes(node, mutant): notes = fit.notes # unicode if notes: - notes = notes[:397] + '...' if len(notes) > 400 else notes + notes = re.sub(r"(\r|\n|\r\n)", "
", notes) + if len(notes) > EVE_FIT_NOTE_MAX: + notes = notes[:EVE_FIT_NOTE_MAX - 3] + '...' - description.setAttribute( - "value", re.sub("(\r|\n|\r\n)+", "
", notes) if notes is not None else "" - ) + description.setAttribute("value", notes) except (KeyboardInterrupt, SystemExit): raise except Exception as e: diff --git a/tests/jeffy_ja-en[99].xml b/tests/jeffy_ja-en[99].xml deleted file mode 100644 index 8ec178b166..0000000000 --- a/tests/jeffy_ja-en[99].xml +++ /dev/null @@ -1,2116 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/tests/localized-fitting-files/ch_ch.xml b/tests/localized-fitting-files/ch_ch.xml new file mode 100644 index 0000000000..5dd498c00a --- /dev/null +++ b/tests/localized-fitting-files/ch_ch.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/localized-fitting-files/ch_en.xml b/tests/localized-fitting-files/ch_en.xml new file mode 100644 index 0000000000..a2dc412277 --- /dev/null +++ b/tests/localized-fitting-files/ch_en.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/localized-fitting-files/de_de.xml b/tests/localized-fitting-files/de_de.xml new file mode 100644 index 0000000000..e6fc111ddf --- /dev/null +++ b/tests/localized-fitting-files/de_de.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/localized-fitting-files/de_en.xml b/tests/localized-fitting-files/de_en.xml new file mode 100644 index 0000000000..1f655af707 --- /dev/null +++ b/tests/localized-fitting-files/de_en.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/localized-fitting-files/es_en.xml b/tests/localized-fitting-files/es_en.xml new file mode 100644 index 0000000000..b412f009ce --- /dev/null +++ b/tests/localized-fitting-files/es_en.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/localized-fitting-files/es_es.xml b/tests/localized-fitting-files/es_es.xml new file mode 100644 index 0000000000..2cd34e1f39 --- /dev/null +++ b/tests/localized-fitting-files/es_es.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/localized-fitting-files/fr_en.xml b/tests/localized-fitting-files/fr_en.xml new file mode 100644 index 0000000000..4700e432e0 --- /dev/null +++ b/tests/localized-fitting-files/fr_en.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/localized-fitting-files/fr_fr.xml b/tests/localized-fitting-files/fr_fr.xml new file mode 100644 index 0000000000..3639e60dee --- /dev/null +++ b/tests/localized-fitting-files/fr_fr.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/localized-fitting-files/ja_en.xml b/tests/localized-fitting-files/ja_en.xml new file mode 100644 index 0000000000..342adf6c8d --- /dev/null +++ b/tests/localized-fitting-files/ja_en.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/localized-fitting-files/ja_ja.xml b/tests/localized-fitting-files/ja_ja.xml new file mode 100644 index 0000000000..fd1f263af9 --- /dev/null +++ b/tests/localized-fitting-files/ja_ja.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/localized-fitting-files/ko_en.xml b/tests/localized-fitting-files/ko_en.xml new file mode 100644 index 0000000000..e1baeef90c --- /dev/null +++ b/tests/localized-fitting-files/ko_en.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/localized-fitting-files/ko_ko.xml b/tests/localized-fitting-files/ko_ko.xml new file mode 100644 index 0000000000..e599480133 --- /dev/null +++ b/tests/localized-fitting-files/ko_ko.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/localized-fitting-files/readme.md b/tests/localized-fitting-files/readme.md new file mode 100644 index 0000000000..f8784c4230 --- /dev/null +++ b/tests/localized-fitting-files/readme.md @@ -0,0 +1,3 @@ +## About the name of the localized fitting file (export from EVE client) + +<EVE client language>_<Important Names language>.xml diff --git a/tests/localized-fitting-files/ru_en.xml b/tests/localized-fitting-files/ru_en.xml new file mode 100644 index 0000000000..52eb9ae4e2 --- /dev/null +++ b/tests/localized-fitting-files/ru_en.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/localized-fitting-files/ru_ru.xml b/tests/localized-fitting-files/ru_ru.xml new file mode 100644 index 0000000000..a9fc6322ca --- /dev/null +++ b/tests/localized-fitting-files/ru_ru.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test_unread_desc.py b/tests/test_unread_desc.py index beb08001a5..873985b849 100644 --- a/tests/test_unread_desc.py +++ b/tests/test_unread_desc.py @@ -72,7 +72,7 @@ def test_import_xml(print_db_info): usr = PortUser() # for path in XML_FILES: xml_file = "jeffy_ja-en[99].xml" - fit_count = int(re.search(r"\[(\d+)\]", xml_file).group(1)) + fit_count = int(re.search(r"\[(\d+)]", xml_file).group(1)) fits = None with open(os.path.join(script_dir, xml_file), "r") as file_: srcString = file_.read() diff --git a/utils/strfunctions.py b/utils/strfunctions.py index 833277d171..f23d307300 100644 --- a/utils/strfunctions.py +++ b/utils/strfunctions.py @@ -5,7 +5,7 @@ def sequential_rep(text_, *args): - # type: (basestring, tuple) -> basestring + # type: (str, list[str]) -> str """ :param text_: string content :param args: like , , , , ... @@ -22,9 +22,9 @@ def sequential_rep(text_, *args): def replace_ltgt(text_): - # type: (basestring) -> basestring + # type: (str) -> str """if fit name contained "<" or ">" then reprace to named html entity by EVE client. :param text_: string content of fit name from exported by EVE client. - :return: if text_ is not instance of basestring then no manipulation to text_. + :return: if text_ is not instance of str then no manipulation to text_. """ return text_.replace("<", "<").replace(">", ">") if isinstance(text_, str) else text_ From a9a90ab747b7ab2952ad83a21794604e1d738a07 Mon Sep 17 00:00:00 2001 From: jeffy-g Date: Sun, 1 Dec 2024 09:46:10 +0900 Subject: [PATCH 02/12] It seems that multi-thread logging works fine if use `applicationbound()` instead of `threadbound()`. --- pyfa.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyfa.py b/pyfa.py index 706af752c3..6664c6b9ed 100755 --- a/pyfa.py +++ b/pyfa.py @@ -110,7 +110,7 @@ def _process_args(self, largs, rargs, values): config.defPaths(options.savepath) config.defLogging() - with config.logging_setup.threadbound(): + with config.logging_setup.applicationbound(): pyfalog.info("Starting Pyfa") pyfalog.info(version_block) From 6aa77e47aa287f93185dd29026e6e1cfbe547f14 Mon Sep 17 00:00:00 2001 From: jeffy-g Date: Sun, 1 Dec 2024 11:45:24 +0900 Subject: [PATCH 03/12] Log whether the fitting is localized --- service/port/xml.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/service/port/xml.py b/service/port/xml.py index fe17ba84c1..918574a885 100644 --- a/service/port/xml.py +++ b/service/port/xml.py @@ -115,7 +115,7 @@ def _solve_ship(fitting, sMkt, b_localized): def _solve_module(hardware, sMkt, b_localized): - # type: (minidom.Element, Market, bool) -> Module + # type: (minidom.Element, Market, bool) -> Item moduleName = hardware.getAttribute("base_type") or hardware.getAttribute("type") emergency = None if b_localized: @@ -173,6 +173,9 @@ def importXml(text, progress): fit_list = [] failed = 0 + pyfalog.info( + f"importXml - localized fitting {'detected' if b_localized else 'is normaly'}" + ) for fitting in fittings: if progress and progress.userCancelled: return [] From f9f0806ba9b65d846833615fb7375df29f81fc24 Mon Sep 17 00:00:00 2001 From: jeffy-g Date: Sun, 1 Dec 2024 13:15:01 +0900 Subject: [PATCH 04/12] service/port/xml.py - refactoring --- eos/saveddata/fit.py | 1 + service/port/xml.py | 106 +++++++++++++++++++++---------------------- 2 files changed, 54 insertions(+), 53 deletions(-) diff --git a/eos/saveddata/fit.py b/eos/saveddata/fit.py index 9abf73e500..4d3eee12bd 100644 --- a/eos/saveddata/fit.py +++ b/eos/saveddata/fit.py @@ -68,6 +68,7 @@ class Fit: PEAK_RECHARGE = 0.25 def __init__(self, ship=None, name=""): + # type: (Ship, str) -> Fit """Initialize a fit from the program""" self.__ship = None self.__mode = None diff --git a/service/port/xml.py b/service/port/xml.py index 918574a885..832bf95fff 100644 --- a/service/port/xml.py +++ b/service/port/xml.py @@ -59,47 +59,64 @@ def _extract_match(t): # hint attribute, text content return m.group(1), m.group(2) - -def _solve_ship(fitting, sMkt, b_localized): - # type: (minidom.Element, Market, bool) -> Fit - """ NOTE: Since it is meaningless unless a correct ship object can be constructed, - process flow changed - """ - # ------ Confirm ship - # Maelstrom - shipType = fitting.getElementsByTagName("shipType")[0].getAttribute("value") - anything = None +def doIt(text, b_localized): + # type: (str, bool) -> tuple[str, str|None] + altText = None if b_localized: try: # expect an official name, emergency cache - shipType, anything = _extract_match(shipType) + text, altText = _extract_match(text) except ExtractingError: pass + return text, altText + +def _solve(name, altName, handler): + # type: (str, str|None, function) -> object|None limit = 2 - ship = None + subject = None while True: must_retry = False try: - try: - ship = Ship(sMkt.getItem(shipType)) - except ValueError: - ship = Citadel(sMkt.getItem(shipType)) + subject = handler(name) except (KeyboardInterrupt, SystemExit): raise except Exception as e: - pyfalog.warning("Caught exception on _solve_ship") - pyfalog.error(e) + # pyfalog.warning("Caught exception on _solve") + pyfalog.error("Caught exception on _solve:: {}", e) limit -= 1 if limit == 0: break - shipType = anything + name = altName must_retry = True if not must_retry: break + return subject + +def _solve_ship(fitting, sMkt, b_localized): + # type: (minidom.Element, Market, bool) -> Fit + """ NOTE: Since it is meaningless unless a correct ship object can be constructed, + process flow changed + """ + def handler(name): + # type: (str) -> Ship + try: + return Ship(sMkt.getItem(name)) + except ValueError: + return Citadel(sMkt.getItem(name)) + + # ------ Confirm ship + # Maelstrom + shipType, anything = doIt( + fitting.getElementsByTagName("shipType")[0].getAttribute("value"), b_localized + ) + ship = _solve(shipType, anything, handler) + if ship is None: - raise Exception("cannot resolve ship type.") + raise Exception( + f"cannot solve ship type, name: '{shipType}', altName: '{anything}'" + ) fitobj = Fit(ship=ship) # ------ Confirm fit name @@ -116,39 +133,22 @@ def _solve_ship(fitting, sMkt, b_localized): def _solve_module(hardware, sMkt, b_localized): # type: (minidom.Element, Market, bool) -> Item - moduleName = hardware.getAttribute("base_type") or hardware.getAttribute("type") - emergency = None - if b_localized: - try: - # expect an official name, emergency cache - moduleName, emergency = _extract_match(moduleName) - except ExtractingError: - pass - - item = None - limit = 2 - while True: - must_retry = False - try: - item = sMkt.getItem(moduleName, eager="group.category") - if not item: - raise ValueError(f"{moduleName} is not valid") - pyfalog.info('_solve_module - sMkt.getItem: {}', item) - except (KeyboardInterrupt, SystemExit): - raise - except Exception as e: - pyfalog.warning("Caught exception on _solve_module, name:{}", moduleName) - pyfalog.error(e) - limit -= 1 - if limit == 0: - break - moduleName = emergency - must_retry = True - if not must_retry and item: - break - + def handler(name): + # type: (str) -> Item + item = sMkt.getItem(name, eager="group.category") + if not item: + raise ValueError(f'"{name}" is not valid') + pyfalog.info('_solve_module - sMkt.getItem: {}', item) + return item + + moduleName, emergency = doIt( + hardware.getAttribute("base_type") or hardware.getAttribute("type"), b_localized + ) + item = _solve(moduleName, emergency, handler) if item is None: - raise Exception("cannot resolve module or item.") + raise Exception( + f"cannot solve module, name: '{moduleName}', altName: '{emergency}'" + ) mutaplasmidName = hardware.getAttribute("mutaplasmid") mutaplasmidItem = fetchItem(mutaplasmidName) if mutaplasmidName else None @@ -174,7 +174,7 @@ def importXml(text, progress): failed = 0 pyfalog.info( - f"importXml - localized fitting {'detected' if b_localized else 'is normaly'}" + f"importXml - fitting is {'localized' if b_localized else 'normally'}" ) for fitting in fittings: if progress and progress.userCancelled: From 5a7acd54ec38e253f72e71568d301a5693caf2f9 Mon Sep 17 00:00:00 2001 From: jeffy-g Date: Sun, 1 Dec 2024 13:42:15 +0900 Subject: [PATCH 05/12] type annotation [skip ci] --- service/port/muta.py | 1 + service/port/port.py | 2 +- service/port/xml.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/service/port/muta.py b/service/port/muta.py index 9c39875d89..e92376c000 100644 --- a/service/port/muta.py +++ b/service/port/muta.py @@ -72,6 +72,7 @@ def parseMutant(lines): def parseMutantAttrs(line): + # type: (str) -> dict[int, float] mutations = {} pairs = [p.strip() for p in line.split(',')] for pair in pairs: diff --git a/service/port/port.py b/service/port/port.py index fde33b06ab..16b68f5686 100644 --- a/service/port/port.py +++ b/service/port/port.py @@ -211,7 +211,7 @@ def importFitFromBuffer(bufferStr, activeFit=None): @classmethod def importAuto(cls, string, path=None, activeFit=None, progress=None): - # type: (str, str, svcFit, object) -> None + # type: (str, str, svcFit, object) -> tuple[str, bool, list[svcFit]] lines = string.splitlines() # Get first line and strip space symbols of it to avoid possible detection errors firstLine = '' diff --git a/service/port/xml.py b/service/port/xml.py index 832bf95fff..f3a6672033 100644 --- a/service/port/xml.py +++ b/service/port/xml.py @@ -132,7 +132,7 @@ def handler(name): def _solve_module(hardware, sMkt, b_localized): - # type: (minidom.Element, Market, bool) -> Item + # type: (minidom.Element, Market, bool) -> tuple[Item, Item|None, dict[int, float]|None] def handler(name): # type: (str) -> Item item = sMkt.getItem(name, eager="group.category") From ff1bd81226c0b4713b18c8b64f855bdeb92d053d Mon Sep 17 00:00:00 2001 From: jeffy-g Date: Sun, 1 Dec 2024 16:39:37 +0900 Subject: [PATCH 06/12] refactoring --- service/port/xml.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/service/port/xml.py b/service/port/xml.py index f3a6672033..cd4b0e7a15 100644 --- a/service/port/xml.py +++ b/service/port/xml.py @@ -296,7 +296,7 @@ def exportXml(fits, progress, callback): fittings = doc.createElement("fittings") # fit count fit_count = len(fits) - fittings.setAttribute("count", "%s" % fit_count) + fittings.setAttribute("count", str(fit_count)) doc.appendChild(fittings) def addMutantAttributes(node, mutant): @@ -310,7 +310,8 @@ def addMutantAttributes(node, mutant): return None processedFits = i + 1 progress.current = processedFits - progress.message = "converting to xml (%s/%s) %s" % (processedFits, fit_count, fit.ship.name) + progress.message = f"converting to xml ({processedFits}/{fit_count}) {fit.ship.name}" + try: fitting = doc.createElement("fitting") fitting.setAttribute("name", fit.name) @@ -318,13 +319,9 @@ def addMutantAttributes(node, mutant): description = doc.createElement("description") # -- 170327 Ignored description -- try: - notes = fit.notes # unicode - - if notes: - notes = re.sub(r"(\r|\n|\r\n)", "
", notes) - if len(notes) > EVE_FIT_NOTE_MAX: - notes = notes[:EVE_FIT_NOTE_MAX - 3] + '...' - + notes = re.sub(r"(\r|\n|\r\n)", "
", fit.notes or "") + if len(notes) > EVE_FIT_NOTE_MAX: + notes = notes[:EVE_FIT_NOTE_MAX - 3] + '...' description.setAttribute("value", notes) except (KeyboardInterrupt, SystemExit): raise @@ -358,7 +355,7 @@ def addMutantAttributes(node, mutant): hardware.setAttribute("type", module.item.name) slotName = FittingSlot(slot).name.lower() slotName = slotName if slotName != "high" else "hi" - hardware.setAttribute("slot", "%s slot %d" % (slotName, slotId)) + hardware.setAttribute("slot", f"{slotName} slot {slotId}") if module.isMutated: addMutantAttributes(hardware, module) @@ -392,16 +389,16 @@ def addMutantAttributes(node, mutant): charges[cargo.item.name] = 0 charges[cargo.item.name] += cargo.amount - for name, qty in list(charges.items()): + for name, qty in charges.items(): hardware = doc.createElement("hardware") - hardware.setAttribute("qty", "%d" % qty) + hardware.setAttribute("qty", str(qty)) hardware.setAttribute("slot", "cargo") hardware.setAttribute("type", name) fitting.appendChild(hardware) except (KeyboardInterrupt, SystemExit): raise except Exception as e: - pyfalog.error("Failed on fitID: %d, message: %s" % e.message) + pyfalog.error(f"Failed on fitID: {fit.ship.ID}, message: {e}") continue text = doc.toprettyxml() From a70d92542de0a62ab9875bab324a83bd39ee87f0 Mon Sep 17 00:00:00 2001 From: jeffy-g Date: Sun, 1 Dec 2024 21:33:56 +0900 Subject: [PATCH 07/12] type annotation --- pyfa.spec | 2 -- service/market.py | 2 +- service/port/xml.py | 37 ++++++++++++++++++++++--------------- utils/strfunctions.py | 2 +- 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/pyfa.spec b/pyfa.spec index 28a219b6da..33fb64e0ad 100644 --- a/pyfa.spec +++ b/pyfa.spec @@ -1,7 +1,5 @@ # -*- mode: python -*- -# If you get a cp65001 error during build, set the environment variable PYTHONIOENCODING="utf8" - import os from itertools import chain import subprocess diff --git a/service/market.py b/service/market.py index 4659f15ef5..cab7255011 100644 --- a/service/market.py +++ b/service/market.py @@ -495,7 +495,7 @@ def __makeReverseMetaMapIndices(self): @staticmethod def getItem(identity, *args, **kwargs): - # type: (str|int|types_Item, list[str], dict[str]) -> types_Item|None + # type: (str|int|types_Item, *str, **str) -> types_Item|None """Get item by its ID or name""" try: if isinstance(identity, types_Item): diff --git a/service/port/xml.py b/service/port/xml.py index cd4b0e7a15..42e57b909d 100644 --- a/service/port/xml.py +++ b/service/port/xml.py @@ -40,6 +40,11 @@ from utils.strfunctions import replace_ltgt, sequential_rep from config import EVE_FIT_NOTE_MAX +from eos.gamedata import Item # for type annotation +# NOTE: I want to define an interface in the utils package and reference it (IProgress) +# gui.utils.progressHelper.ProgressHelper inherits utils.IProgress +from gui.utils.progressHelper import ProgressHelper # for type annotation + pyfalog = Logger(__name__) @@ -72,7 +77,7 @@ def doIt(text, b_localized): return text, altText def _solve(name, altName, handler): - # type: (str, str|None, function) -> object|None + # type: (str, str|None, function) -> any # enable inferer limit = 2 subject = None while True: @@ -101,17 +106,18 @@ def _solve_ship(fitting, sMkt, b_localized): """ def handler(name): # type: (str) -> Ship + item = sMkt.getItem(name) try: - return Ship(sMkt.getItem(name)) + return Ship(item) except ValueError: - return Citadel(sMkt.getItem(name)) + return Citadel(item) # ------ Confirm ship # Maelstrom shipType, anything = doIt( fitting.getElementsByTagName("shipType")[0].getAttribute("value"), b_localized ) - ship = _solve(shipType, anything, handler) + ship = _solve(shipType, anything, handler) # type: Ship if ship is None: raise Exception( @@ -122,7 +128,7 @@ def handler(name): # ------ Confirm fit name anything = fitting.getAttribute("name") # 2017/03/29 NOTE: - # if fit name contained "<" or ">" then reprace to named html entity by EVE client + # if fit name contained "<" or ">" then replace to named html entity by EVE client # if re.search(RE_LTGT, anything): if "<" in anything or ">" in anything: anything = replace_ltgt(anything) @@ -135,11 +141,11 @@ def _solve_module(hardware, sMkt, b_localized): # type: (minidom.Element, Market, bool) -> tuple[Item, Item|None, dict[int, float]|None] def handler(name): # type: (str) -> Item - item = sMkt.getItem(name, eager="group.category") - if not item: + mod = sMkt.getItem(name, eager="group.category") + if not mod: raise ValueError(f'"{name}" is not valid') - pyfalog.info('_solve_module - sMkt.getItem: {}', item) - return item + pyfalog.info('_solve_module - sMkt.getItem: {}', mod) + return mod moduleName, emergency = doIt( hardware.getAttribute("base_type") or hardware.getAttribute("type"), b_localized @@ -160,17 +166,16 @@ def handler(name): def importXml(text, progress): - # type: (str, object) -> list[Fit] + # type: (str, ProgressHelper) -> list[Fit] from .port import Port sMkt = Market.getInstance() - doc = minidom.parseString(text) # NOTE: # When L_MARK is included at this point, # Decided to be localized data b_localized = L_MARK in text - fittings = doc.getElementsByTagName("fitting") - fit_list = [] + fittings = minidom.parseString(text).getElementsByTagName("fitting") + fit_list = [] # type: list[Fit] failed = 0 pyfalog.info( @@ -204,7 +209,7 @@ def importXml(text, progress): hardwares = fitting.getElementsByTagName("hardware") # Sorting by "slot" attr is cool hardwares.sort(key=lambda e: e.getAttribute("slot")) - moduleList = [] + moduleList = [] # type: list[Module] for hardware in hardwares: try: item, mutaItem, mutaAttrs = _solve_module(hardware, sMkt, b_localized) @@ -240,7 +245,7 @@ def importXml(text, progress): c.amount = int(hardware.getAttribute("qty")) fitobj.cargo.append(c) else: - m = None + m = None # type: Module try: if mutaItem: mutaplasmid = getDynamicItem(mutaItem.ID) @@ -288,6 +293,8 @@ def importXml(text, progress): if progress: progress.message = "Processing %s\n%s" % (fitobj.ship.name, fitobj.name) + pyfalog.info(f"importXml - stats of parse, succeeded: {fittings.length - failed}, failed: {failed}") + return fit_list def exportXml(fits, progress, callback): diff --git a/utils/strfunctions.py b/utils/strfunctions.py index f23d307300..dbbc525c91 100644 --- a/utils/strfunctions.py +++ b/utils/strfunctions.py @@ -5,7 +5,7 @@ def sequential_rep(text_, *args): - # type: (str, list[str]) -> str + # type: (str, *str) -> str """ :param text_: string content :param args: like , , , , ... From 02a7958a99992cf34e767e3eb1457605ace859dd Mon Sep 17 00:00:00 2001 From: jeffy-g Date: Mon, 2 Dec 2024 05:42:05 +0900 Subject: [PATCH 08/12] using `html.unescape` instead of `replace_ltgt` etc --- service/port/esi.py | 2 +- service/port/xml.py | 7 ++++--- utils/strfunctions.py | 10 ---------- 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/service/port/esi.py b/service/port/esi.py index ef708c9726..1bf4ed7988 100644 --- a/service/port/esi.py +++ b/service/port/esi.py @@ -63,7 +63,7 @@ def exportESI(ofit, exportCharges, exportImplants, exportBoosters, callback): nested_dict = lambda: defaultdict(nested_dict) fit = nested_dict() - sFit = svcFit.getInstance() + # sFit = svcFit.getInstance() # max length is 50 characters name = ofit.name[:47] + '...' if len(ofit.name) > 50 else ofit.name diff --git a/service/port/xml.py b/service/port/xml.py index 42e57b909d..1e2f798184 100644 --- a/service/port/xml.py +++ b/service/port/xml.py @@ -37,7 +37,8 @@ from service.market import Market from service.port.muta import renderMutantAttrs, parseMutantAttrs from service.port.shared import fetchItem -from utils.strfunctions import replace_ltgt, sequential_rep +from html import unescape +from utils.strfunctions import sequential_rep from config import EVE_FIT_NOTE_MAX from eos.gamedata import Item # for type annotation @@ -131,7 +132,7 @@ def handler(name): # if fit name contained "<" or ">" then replace to named html entity by EVE client # if re.search(RE_LTGT, anything): if "<" in anything or ">" in anything: - anything = replace_ltgt(anything) + anything = unescape(anything) fitobj.name = anything return fitobj @@ -201,7 +202,7 @@ def importXml(text, progress): elif len(description): # convert
to "\n" and remove html tags. if Port.is_tag_replace(): - description = replace_ltgt( + description = unescape( sequential_rep(description, r"<(br|BR)>", "\n", r"<[^<>]+>", "") ) fitobj.notes = description diff --git a/utils/strfunctions.py b/utils/strfunctions.py index dbbc525c91..5a3bcdd982 100644 --- a/utils/strfunctions.py +++ b/utils/strfunctions.py @@ -3,7 +3,6 @@ """ import re - def sequential_rep(text_, *args): # type: (str, *str) -> str """ @@ -19,12 +18,3 @@ def sequential_rep(text_, *args): i += 2 return text_ - - -def replace_ltgt(text_): - # type: (str) -> str - """if fit name contained "<" or ">" then reprace to named html entity by EVE client. - :param text_: string content of fit name from exported by EVE client. - :return: if text_ is not instance of str then no manipulation to text_. - """ - return text_.replace("<", "<").replace(">", ">") if isinstance(text_, str) else text_ From 5dec17db7c12216d1a882e4e1f81267443df9f25 Mon Sep 17 00:00:00 2001 From: jeffy-g Date: Mon, 2 Dec 2024 10:23:44 +0900 Subject: [PATCH 09/12] temporary commit --- gui/mainFrame.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gui/mainFrame.py b/gui/mainFrame.py index 169d31b4a4..60dec4273c 100644 --- a/gui/mainFrame.py +++ b/gui/mainFrame.py @@ -927,8 +927,8 @@ def handleProgress(self, title, style, call, progress, errMsgLbl=None): func, args, kwargs = call func(*args, **kwargs) while progress.working: - wx.MilliSleep(250) - wx.Yield() + wx.MilliSleep(1) + # wx.Yield() (progress.dlgWorking, skip) = dlg.Update(progress.current, progress.message) if progress.error and errMsgLbl: with wx.MessageDialog( From d983e2a784e8e003ed9d2b5c4641cc0a4a3d3006 Mon Sep 17 00:00:00 2001 From: jeffy-g Date: Mon, 2 Dec 2024 13:30:36 +0900 Subject: [PATCH 10/12] Progress Processing under review (xml import only --- gui/mainFrame.py | 17 +++++--- gui/utils/progressHelper.py | 19 ++++++-- service/port/port.py | 13 +++--- service/port/xml.py | 3 +- tests/test_unread_desc.py | 86 ------------------------------------- 5 files changed, 38 insertions(+), 100 deletions(-) delete mode 100644 tests/test_unread_desc.py diff --git a/gui/mainFrame.py b/gui/mainFrame.py index 60dec4273c..11e3d5dc9b 100644 --- a/gui/mainFrame.py +++ b/gui/mainFrame.py @@ -851,7 +851,8 @@ def fileImportDialog(self, event): call = (Port.importFitsThreaded, [dlg.GetPaths(), progress], {}) self.handleProgress( title=_t("Importing fits"), - style=wx.PD_CAN_ABORT | wx.PD_SMOOTH | wx.PD_APP_MODAL | wx.PD_AUTO_HIDE, + # style=wx.PD_CAN_ABORT | wx.PD_SMOOTH | wx.PD_APP_MODAL | wx.PD_AUTO_HIDE, + style=wx.PD_CAN_ABORT | wx.PD_SMOOTH | wx.PD_ELAPSED_TIME | wx.PD_APP_MODAL, # old style at 2017 call=call, progress=progress, errMsgLbl=_t("Import Error")) @@ -914,7 +915,8 @@ def exportHtml(self, event): progress=progress) def handleProgress(self, title, style, call, progress, errMsgLbl=None): - extraArgs = {} + # type: (str, int, tuple[function, list[str], dict[str, str]], ProgressHelper, str) -> None + extraArgs = {} # type: dict[str, any] if progress.maximum is not None: extraArgs['maximum'] = progress.maximum with wx.ProgressDialog( @@ -925,11 +927,16 @@ def handleProgress(self, title, style, call, progress, errMsgLbl=None): **extraArgs ) as dlg: func, args, kwargs = call + # IMPORTANT + progress.dlg = dlg func(*args, **kwargs) while progress.working: - wx.MilliSleep(1) - # wx.Yield() - (progress.dlgWorking, skip) = dlg.Update(progress.current, progress.message) + wx.MilliSleep(20) + wx.Yield() + # (progress.dlgWorking, skip) = dlg.Pulse( + # # progress.current, + # progress.message + # ) if progress.error and errMsgLbl: with wx.MessageDialog( self, diff --git a/gui/utils/progressHelper.py b/gui/utils/progressHelper.py index cda25b55dc..472c8b7c45 100644 --- a/gui/utils/progressHelper.py +++ b/gui/utils/progressHelper.py @@ -1,14 +1,27 @@ +from wx import ProgressDialog class ProgressHelper: def __init__(self, message, maximum=None, callback=None): + #type: (str, int, function) -> None self.message = message self.current = 0 self.maximum = maximum - self.workerWorking = True - self.dlgWorking = True - self.error = None + self.workerWorking = True # type: bool + self.dlgWorking = True # type: bool + self.error = None # type: str self.callback = callback self.cbArgs = [] + self.dlg = None # type: ProgressDialog + + def pulse(self, msg): + # type: (str) -> None + if (self.dlg): + self.dlgWorking, skip = self.dlg.Pulse(msg) + + def update(self, value, msg): + # type: (int, str) -> None + if (self.dlg): + self.dlgWorking, skip = self.dlg.Update(value, msg) @property def working(self): diff --git a/service/port/port.py b/service/port/port.py index 16b68f5686..a83f59b7fc 100644 --- a/service/port/port.py +++ b/service/port/port.py @@ -42,6 +42,7 @@ from service.port.xml import importXml, exportXml from service.port.muta import parseMutant, parseDynamicItemString, fetchDynamicItem +from gui.utils.progressHelper import ProgressHelper # for type annotation pyfalog = Logger(__name__) @@ -100,7 +101,7 @@ def backupFitsWorkerFunc(path, progress): @staticmethod def importFitsThreaded(paths, progress): - # type: (list[str], object) -> None + # type: (list[str], ProgressHelper) -> None """ :param paths: fits data file path list. :rtype: None @@ -113,6 +114,7 @@ def importFitsThreaded(paths, progress): @staticmethod def importFitFromFiles(paths, progress=None): + # type: (list[str], ProgressHelper) -> tuple[bool, list[svcFit]] """ Imports fits from file(s). First processes all provided paths and stores assembled fits into a list. This allows us to call back to the GUI as @@ -122,7 +124,7 @@ def importFitFromFiles(paths, progress=None): sFit = svcFit.getInstance() - fit_list = [] + fit_list = [] # type: list[svcFit] try: for path in paths: if progress: @@ -130,7 +132,7 @@ def importFitFromFiles(paths, progress=None): progress.workerWorking = False return False, "Cancelled by user" msg = "Processing file:\n%s" % path - progress.message = msg + progress.pulse(msg) pyfalog.debug(msg) with open(path, "rb") as file_: @@ -171,7 +173,8 @@ def importFitFromFiles(paths, progress=None): # IDs.append(fit.ID) if progress: pyfalog.debug("Processing complete, saving fits to database: {0}/{1}", idx + 1, numFits) - progress.message = "Processing complete, saving fits to database\n(%d/%d) %s" % (idx + 1, numFits, fit.ship.name) + # progress.message = "Processing complete, saving fits to database\n(%d/%d) %s" % (idx + 1, numFits, fit.ship.name) + progress.pulse(f"Processing complete, saving fits to database\n({idx + 1}/{numFits}) {fit.ship.name}") except (KeyboardInterrupt, SystemExit): raise except Exception as e: @@ -211,7 +214,7 @@ def importFitFromBuffer(bufferStr, activeFit=None): @classmethod def importAuto(cls, string, path=None, activeFit=None, progress=None): - # type: (str, str, svcFit, object) -> tuple[str, bool, list[svcFit]] + # type: (str, str, svcFit, ProgressHelper) -> tuple[str, bool, list[svcFit]] lines = string.splitlines() # Get first line and strip space symbols of it to avoid possible detection errors firstLine = '' diff --git a/service/port/xml.py b/service/port/xml.py index 1e2f798184..8f20f9db19 100644 --- a/service/port/xml.py +++ b/service/port/xml.py @@ -292,7 +292,8 @@ def importXml(text, progress): fit_list.append(fitobj) if progress: - progress.message = "Processing %s\n%s" % (fitobj.ship.name, fitobj.name) + progress.pulse(f"Processing {fitobj.ship.name}\n{fitobj.name}") + # progress.message = "Processing %s\n%s" % (fitobj.ship.name, fitobj.name) pyfalog.info(f"importXml - stats of parse, succeeded: {fittings.length - failed}, failed: {failed}") diff --git a/tests/test_unread_desc.py b/tests/test_unread_desc.py deleted file mode 100644 index 873985b849..0000000000 --- a/tests/test_unread_desc.py +++ /dev/null @@ -1,86 +0,0 @@ -""" - 2017/04/05: unread description tests module. -""" -# noinspection PyPackageRequirements -import pytest -# Add root folder to python paths -# This must be done on every test in order to pass in Travis -import os -import sys -# nopep8 -import re - -script_dir = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(os.path.realpath(os.path.join(script_dir, '..'))) -sys._called_from_test = True # need db open for tests. (see eos/config.py#17 - -# This import is here to hack around circular import issues -import gui.mainFrame -# noinspection PyPep8 -from service.port import Port, IPortUser - -""" -NOTE: - description character length is restricted 4hundred by EVE client. - these things apply to multi byte environment too. - - - o read xml fit data (and encode to utf-8 if need. - - o construct xml dom object, and extract "fitting" elements. - - o apply _resolve_ship method to each "fitting" elements. (time measurement - - o extract "hardware" elements from "fitting" element. - - o apply _resolve_module method to each "hardware" elements. (time measurement - -xml files: - "jeffy_ja-en[99].xml" - -NOTE of @decorator: - o Function to receive arguments of function to be decorated - o A function that accepts the decorate target function itself as an argument - o A function that accepts arguments of the decorator itself - -for local coverage: - py.test --cov=./ --cov-report=html -""" - -class PortUser(IPortUser): - - def on_port_processing(self, action, data=None): - print(data) - return True - - -#stpw = Stopwatch('test measurementer') - -@pytest.fixture() -def print_db_info(): - # Output debug info - import eos - print() - print("------------ data base connection info ------------") - print(eos.db.saveddata_engine) - print(eos.db.gamedata_engine) - print() - - -# noinspection PyUnusedLocal -def test_import_xml(print_db_info): - usr = PortUser() -# for path in XML_FILES: - xml_file = "jeffy_ja-en[99].xml" - fit_count = int(re.search(r"\[(\d+)]", xml_file).group(1)) - fits = None - with open(os.path.join(script_dir, xml_file), "r") as file_: - srcString = file_.read() - srcString = str(srcString, "utf-8") - # (basestring, IPortUser, basestring) -> list[eos.saveddata.fit.Fit] - usr.on_port_process_start() - #stpw.reset() - #with stpw: - fits = Port.importXml(srcString, usr) - - assert fits is not None and len(fits) is fit_count From d9f661c66933b9f1007056cf9bcdafcf40ef9fd6 Mon Sep 17 00:00:00 2001 From: jeffy-g Date: Mon, 2 Dec 2024 15:32:00 +0900 Subject: [PATCH 11/12] maybe done --- eos/db/saveddata/queries.py | 1 + gui/mainFrame.py | 13 +++++-------- gui/utils/progressHelper.py | 24 ++++++++++++++++-------- service/port/port.py | 26 +++++++++++++++----------- service/port/xml.py | 36 ++++++++++++++++++++++++------------ 5 files changed, 61 insertions(+), 39 deletions(-) diff --git a/eos/db/saveddata/queries.py b/eos/db/saveddata/queries.py index 42d5263d11..aa10fe267e 100644 --- a/eos/db/saveddata/queries.py +++ b/eos/db/saveddata/queries.py @@ -280,6 +280,7 @@ def getFitsWithModules(typeIDs, eager=None): def countAllFits(): + # type: () -> int with sd_lock: count = saveddata_session.query(Fit).count() return count diff --git a/gui/mainFrame.py b/gui/mainFrame.py index 11e3d5dc9b..790909badd 100644 --- a/gui/mainFrame.py +++ b/gui/mainFrame.py @@ -852,7 +852,7 @@ def fileImportDialog(self, event): self.handleProgress( title=_t("Importing fits"), # style=wx.PD_CAN_ABORT | wx.PD_SMOOTH | wx.PD_APP_MODAL | wx.PD_AUTO_HIDE, - style=wx.PD_CAN_ABORT | wx.PD_SMOOTH | wx.PD_ELAPSED_TIME | wx.PD_APP_MODAL, # old style at 2017 + style=wx.PD_CAN_ABORT | wx.PD_SMOOTH | wx.PD_ELAPSED_TIME | wx.PD_APP_MODAL | wx.PD_AUTO_HIDE, # old style at 2017 | wx.RESIZE_BORDER call=call, progress=progress, errMsgLbl=_t("Import Error")) @@ -868,7 +868,7 @@ def backupToXml(self, event): style=wx.FD_SAVE | wx.FD_OVERWRITE_PROMPT, defaultFile=defaultFile) as fileDlg: if fileDlg.ShowModal() == wx.ID_OK: - filePath = fileDlg.GetPath() + filePath = fileDlg.GetPath() # type: str if '.' not in os.path.basename(filePath): filePath += ".xml" @@ -927,16 +927,13 @@ def handleProgress(self, title, style, call, progress, errMsgLbl=None): **extraArgs ) as dlg: func, args, kwargs = call - # IMPORTANT + # important progress.dlg = dlg func(*args, **kwargs) while progress.working: - wx.MilliSleep(20) + wx.MilliSleep(33) wx.Yield() - # (progress.dlgWorking, skip) = dlg.Pulse( - # # progress.current, - # progress.message - # ) + (progress.dlgWorking, skip) = dlg.Update(progress.current, progress.message) if progress.error and errMsgLbl: with wx.MessageDialog( self, diff --git a/gui/utils/progressHelper.py b/gui/utils/progressHelper.py index 472c8b7c45..b67cd105b2 100644 --- a/gui/utils/progressHelper.py +++ b/gui/utils/progressHelper.py @@ -10,18 +10,26 @@ def __init__(self, message, maximum=None, callback=None): self.dlgWorking = True # type: bool self.error = None # type: str self.callback = callback - self.cbArgs = [] + self.cbArgs = [] # type: list[str] self.dlg = None # type: ProgressDialog - def pulse(self, msg): - # type: (str) -> None + def setRange(self, max): + # type: (int) -> None + """ + call ProgressDialog.SetRange(max) + """ if (self.dlg): - self.dlgWorking, skip = self.dlg.Pulse(msg) + self.dlg.SetRange(max) - def update(self, value, msg): - # type: (int, str) -> None - if (self.dlg): - self.dlgWorking, skip = self.dlg.Update(value, msg) + # def pulse(self, msg): + # # type: (str) -> None + # if (self.dlg): + # self.dlgWorking, skip = self.dlg.Pulse(msg) + + # def update(self, value, msg): + # # type: (int, str) -> None + # if (self.dlg): + # self.dlgWorking, skip = self.dlg.Update(value, msg) @property def working(self): diff --git a/service/port/port.py b/service/port/port.py index a83f59b7fc..7ef9681ce9 100644 --- a/service/port/port.py +++ b/service/port/port.py @@ -74,11 +74,11 @@ def is_tag_replace(cls): @staticmethod def backupFits(path, progress): - # type: (str, object) -> None + # type: (str, ProgressHelper) -> None pyfalog.debug("Starting backup fits thread.") def backupFitsWorkerFunc(path, progress): - # type: (str, object) -> None + # type: (str, ProgressHelper) -> None try: backedUpFits = Port.exportXml(svcFit.getInstance().getAllFits(), progress) if backedUpFits: @@ -131,8 +131,8 @@ def importFitFromFiles(paths, progress=None): if progress.userCancelled: progress.workerWorking = False return False, "Cancelled by user" - msg = "Processing file:\n%s" % path - progress.pulse(msg) + msg = f"Processing file: {path}" + progress.message = msg pyfalog.debug(msg) with open(path, "rb") as file_: @@ -156,10 +156,14 @@ def importFitFromFiles(paths, progress=None): return False, msg numFits = len(fit_list) + progress.setRange(numFits) for idx, fit in enumerate(fit_list): - if progress and progress.userCancelled: - progress.workerWorking = False - return False, "Cancelled by user" + if progress: + if (progress.userCancelled): + progress.workerWorking = False + return False, "Cancelled by user" + + progress.current = idx + 1 # Set some more fit attributes and save fit.character = sFit.character fit.damagePattern = sFit.pattern @@ -172,9 +176,9 @@ def importFitFromFiles(paths, progress=None): db.save(fit) # IDs.append(fit.ID) if progress: - pyfalog.debug("Processing complete, saving fits to database: {0}/{1}", idx + 1, numFits) - # progress.message = "Processing complete, saving fits to database\n(%d/%d) %s" % (idx + 1, numFits, fit.ship.name) - progress.pulse(f"Processing complete, saving fits to database\n({idx + 1}/{numFits}) {fit.ship.name}") + msg = "Processing complete, saving fits to database" + pyfalog.debug(f"{msg}: {idx + 1}/{numFits}") + progress.message = f"{msg}\n({idx + 1}/{numFits}) {fit.ship.name}" except (KeyboardInterrupt, SystemExit): raise except Exception as e: @@ -226,7 +230,7 @@ def importAuto(cls, string, path=None, activeFit=None, progress=None): # If XML-style start of tag encountered, detect as XML if re.search(RE_XML_START, firstLine): - return "XML", True, cls.importXml(string, progress) + return "XML", True, importXml(string, progress, path) # If JSON-style start, parse as CREST/JSON if firstLine[0] == '{': diff --git a/service/port/xml.py b/service/port/xml.py index 8f20f9db19..9c78f971fc 100644 --- a/service/port/xml.py +++ b/service/port/xml.py @@ -50,7 +50,6 @@ pyfalog = Logger(__name__) # -- 170327 Ignored description -- -RE_LTGT = "&(lt|gt);" L_MARK = "<localized hint="" # <localized hint="([^"]+)">([^\*]+)\*<\/localized> LOCALIZED_PATTERN = re.compile(r'([^*]+)\*?') @@ -130,8 +129,7 @@ def handler(name): anything = fitting.getAttribute("name") # 2017/03/29 NOTE: # if fit name contained "<" or ">" then replace to named html entity by EVE client - # if re.search(RE_LTGT, anything): - if "<" in anything or ">" in anything: + if re.search(f"&(lt|gt);", anything): anything = unescape(anything) fitobj.name = anything @@ -166,9 +164,10 @@ def handler(name): return item, mutaplasmidItem, mutatedAttrs -def importXml(text, progress): - # type: (str, ProgressHelper) -> list[Fit] +def importXml(text, progress, path="---"): + # type: (str, ProgressHelper, str) -> list[Fit] from .port import Port + import os.path sMkt = Market.getInstance() # NOTE: @@ -182,9 +181,15 @@ def importXml(text, progress): pyfalog.info( f"importXml - fitting is {'localized' if b_localized else 'normally'}" ) - for fitting in fittings: - if progress and progress.userCancelled: - return [] + + progress.maximum = fittings.length + progress.setRange(fittings.length) + progress.current = 0 + path = os.path.basename(path) + for idx, fitting in enumerate(fittings): + if progress: + if (progress.userCancelled): + return [] try: fitobj = _solve_ship(fitting, sMkt, b_localized) @@ -194,6 +199,16 @@ def importXml(text, progress): failed += 1 continue + if progress: + currentIdx = idx + 1 + if (currentIdx < fittings.length): + progress.current = currentIdx + # progress.pulse(f"Processing {fitobj.ship.name}\n{fitobj.name}") + progress.message = f"""Processing file: {path} + current - {fitobj.ship.name} + fit name - {fitobj.name} +""" + # progress.message = "Processing %s\n%s" % (fitobj.ship.name, fitobj.name) # -- 170327 Ignored description -- # read description from exported xml. (EVE client, EFT) description = fitting.getElementsByTagName("description")[0].getAttribute("value") @@ -291,16 +306,13 @@ def importXml(text, progress): fitobj.modules.append(module) fit_list.append(fitobj) - if progress: - progress.pulse(f"Processing {fitobj.ship.name}\n{fitobj.name}") - # progress.message = "Processing %s\n%s" % (fitobj.ship.name, fitobj.name) pyfalog.info(f"importXml - stats of parse, succeeded: {fittings.length - failed}, failed: {failed}") return fit_list def exportXml(fits, progress, callback): - # type: (list[Fit], object, any) -> str|None + # type: (list[Fit], ProgressHelper, function) -> str|None doc = minidom.Document() fittings = doc.createElement("fittings") # fit count From a5e13fb84efd1600cb426cf8900205c133051ff8 Mon Sep 17 00:00:00 2001 From: jeffy-g Date: Fri, 6 Dec 2024 11:33:50 +0900 Subject: [PATCH 12/12] small fixes --- gui/utils/progressHelper.py | 1 + service/port/port.py | 3 ++- service/port/xml.py | 16 +++++++--------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/gui/utils/progressHelper.py b/gui/utils/progressHelper.py index b67cd105b2..2c2737727d 100644 --- a/gui/utils/progressHelper.py +++ b/gui/utils/progressHelper.py @@ -18,6 +18,7 @@ def setRange(self, max): """ call ProgressDialog.SetRange(max) """ + self.maximum = max if (self.dlg): self.dlg.SetRange(max) diff --git a/service/port/port.py b/service/port/port.py index 7ef9681ce9..9d69a288fd 100644 --- a/service/port/port.py +++ b/service/port/port.py @@ -156,7 +156,8 @@ def importFitFromFiles(paths, progress=None): return False, msg numFits = len(fit_list) - progress.setRange(numFits) + if progress: + progress.setRange(numFits) for idx, fit in enumerate(fit_list): if progress: if (progress.userCancelled): diff --git a/service/port/xml.py b/service/port/xml.py index 9c78f971fc..f93f81277d 100644 --- a/service/port/xml.py +++ b/service/port/xml.py @@ -182,14 +182,13 @@ def importXml(text, progress, path="---"): f"importXml - fitting is {'localized' if b_localized else 'normally'}" ) - progress.maximum = fittings.length - progress.setRange(fittings.length) - progress.current = 0 - path = os.path.basename(path) + if progress: + progress.setRange(fittings.length) + progress.current = 0 + path = os.path.basename(path) for idx, fitting in enumerate(fittings): - if progress: - if (progress.userCancelled): - return [] + if progress and progress.userCancelled: + return [] try: fitobj = _solve_ship(fitting, sMkt, b_localized) @@ -203,12 +202,11 @@ def importXml(text, progress, path="---"): currentIdx = idx + 1 if (currentIdx < fittings.length): progress.current = currentIdx - # progress.pulse(f"Processing {fitobj.ship.name}\n{fitobj.name}") + # progress.message = "Processing %s\n%s" % (fitobj.ship.name, fitobj.name) progress.message = f"""Processing file: {path} current - {fitobj.ship.name} fit name - {fitobj.name} """ - # progress.message = "Processing %s\n%s" % (fitobj.ship.name, fitobj.name) # -- 170327 Ignored description -- # read description from exported xml. (EVE client, EFT) description = fitting.getElementsByTagName("description")[0].getAttribute("value")