From 1ff9a668e7b775544e2054aaadd9c3105c911d68 Mon Sep 17 00:00:00 2001 From: robertsaternus Date: Tue, 19 May 2026 15:51:16 +0200 Subject: [PATCH 1/3] Export flat records - testing --- src/Command/DataExportCommand.php | 3 + .../Data/Factory/ProductEntityFactory.php | 31 +++++----- src/Export/ExportProducts.php | 57 +++++++++++++++---- src/Export/Feed.php | 7 +++ src/Export/Field/FilterAttributes.php | 38 +++++++++---- 5 files changed, 101 insertions(+), 35 deletions(-) diff --git a/src/Command/DataExportCommand.php b/src/Command/DataExportCommand.php index 591a4808..4d8af710 100644 --- a/src/Command/DataExportCommand.php +++ b/src/Command/DataExportCommand.php @@ -154,6 +154,9 @@ public function execute(InputInterface $input, OutputInterface $output): int unlink(stream_get_meta_data($this->file)['uri']); } + $peakMemory = memory_get_peak_usage(true) / 1024 / 1024; + $output->writeln(sprintf('Peak Memory Usage: %.2f MB', $peakMemory)); + return 0; } diff --git a/src/Export/Data/Factory/ProductEntityFactory.php b/src/Export/Data/Factory/ProductEntityFactory.php index 1528f2db..bf4cb58e 100644 --- a/src/Export/Data/Factory/ProductEntityFactory.php +++ b/src/Export/Data/Factory/ProductEntityFactory.php @@ -23,7 +23,7 @@ public function __construct( PropertyFormatter $propertyFormatter, FieldsProvider $fieldsProviders, CurrencyFieldsProvider $currencyFieldsProvider, - \Traversable $variantFields, + \Traversable $variantFields ) { $this->propertyFormatter = $propertyFormatter; $this->fieldsProvider = $fieldsProviders; @@ -37,22 +37,27 @@ public function handle(Entity $entity): bool } /** - * @param Entity $entity - * @param string $producedType - * - * @return ProductEntity[]|iterable - * - * @SuppressWarnings(PHPMD.UnusedFormalParameter) + * @param SalesChannelProductEntity $entity */ public function createEntities(Entity $entity, string $producedType = ProductEntity::class): iterable { - // @todo use spread operator? $fields = array_merge($this->fieldsProvider->getFields($producedType), $this->currencyFieldsProvider->getCurrencyFields()); - $parent = new $producedType($entity, new \ArrayIterator($fields), new \ArrayIterator()); - if ($entity->getChildCount()) { - yield from $entity->getChildren()->map(fn ( - SalesChannelProductEntity $child) => new VariantEntity($child, $parent->toArray(), $this->propertyFormatter, iterator_to_array($this->variantFields))); + + if ($entity->getParentId() !== null) { + // To jest WARIANT. + // Tworzymy bazową encję ProductEntity (odpowiednik rodzica), ponieważ wariant w + // Shopware posiada w sobie dziedziczone dane od rodzica (nazwa, opis, producent). + $pseudoParent = new $producedType($entity, new \ArrayIterator($fields), new \ArrayIterator()); + + yield new VariantEntity( + $entity, + $pseudoParent->toArray(), + $this->propertyFormatter, + iterator_to_array($this->variantFields) + ); + } else { + // To jest PRODUKT GŁÓWNY (Rodzic lub samodzielny produkt). + yield new $producedType($entity, new \ArrayIterator($fields), new \ArrayIterator()); } - yield $parent; } } diff --git a/src/Export/ExportProducts.php b/src/Export/ExportProducts.php index 974e7551..c040a9e0 100644 --- a/src/Export/ExportProducts.php +++ b/src/Export/ExportProducts.php @@ -25,12 +25,39 @@ public function __construct(SalesChannelRepository $productRepository, array $cu public function getByContext(SalesChannelContext $context, int $batchSize = 100): iterable { - $criteria = $this->getCriteria($batchSize); - $products = $this->productRepository->search($criteria, $context); - while ($products->count()) { - yield from $products; - $criteria->setOffset($criteria->getOffset() + $criteria->getLimit()); + $offset = 0; + + while (true) { + $criteria = $this->getCriteria($batchSize, $offset); $products = $this->productRepository->search($criteria, $context); + + if ($products->count() === 0) { + break; + } + + foreach ($products->getElements() as $product) { + yield $product; + } + + $products->clear(); + unset($products, $criteria); + gc_collect_cycles(); + + // --- DEBUG PAMIĘCI START --- + // Przeliczamy bajty na megabajty dla czytelności + $memoryUsageMB = memory_get_usage(true) / 1024 / 1024; + $peakMemoryMB = memory_get_peak_usage(true) / 1024 / 1024; + + echo sprintf( + "[%s] Offset: %d | Memory: %.2f MB | Peak: %.2f MB\n", + date('H:i:s'), + $offset, + $memoryUsageMB, + $peakMemoryMB + ); + // --- DEBUG PAMIĘCI END --- + + $offset += $batchSize; } } @@ -39,24 +66,30 @@ public function getProducedExportEntityType(): string return ExportProductEntity::class; } - private function getCriteria(int $batchSize): Criteria + private function getCriteria(int $batchSize, int $offset): Criteria { $criteria = new Criteria(); $criteria->setLimit($batchSize); + $criteria->setOffset($offset); $criteria->addAssociation('categories'); $criteria->addAssociation('categoriesRo'); - $criteria->addAssociation('children.options.group'); $criteria->addAssociation('manufacturer'); - $criteria->addAssociation('properties'); - $criteria->addAssociation('customFields'); $criteria->addAssociation('properties.group'); - $criteria->addAssociation('seoUrls'); + $criteria->addAssociation('options.group'); $criteria->addAssociation('media'); - $criteria->addAssociation('children.cover.media'); + $criteria->addAssociation('cover.media'); + $criteria->addAssociation('seoUrls'); + $criteria->addAssociation('customFields'); + + $criteria->addAssociation('configuratorSettings.option.group'); + foreach ($this->customAssociations as $association) { $criteria->addAssociation($association); } - $criteria->addFilter(new EqualsFilter('parentId', null)); + + // UWAGA: Usunęliśmy addFilter(new EqualsFilter('parentId', null)); + // Chcemy eksportować płasko wszystko: zarówno rodziców jak i warianty, + // więc nie filtrujemy tutaj po parentId! return $criteria; } diff --git a/src/Export/Feed.php b/src/Export/Feed.php index b9047885..959a08b6 100644 --- a/src/Export/Feed.php +++ b/src/Export/Feed.php @@ -29,9 +29,16 @@ public function generate(StreamInterface $stream, array $columns): void $stream->addEntity($columns); $emptyRecord = array_combine($columns, array_fill(0, count($columns), '')); + $i = 0; + foreach ($this->getEntities() as $entity) { $entityData = array_merge($emptyRecord, array_intersect_key($entity->toArray(), $emptyRecord)); $stream->addEntity($this->prepare($entityData)); + + unset($entity, $entityData); + if (++$i % 500 === 0) { + gc_collect_cycles(); + } } } diff --git a/src/Export/Field/FilterAttributes.php b/src/Export/Field/FilterAttributes.php index 3f977795..d6b0cdd0 100644 --- a/src/Export/Field/FilterAttributes.php +++ b/src/Export/Field/FilterAttributes.php @@ -29,16 +29,33 @@ public function getName(): string /** * @param Product $entity - * - * @return string */ public function getValue(Entity $entity): string { - $attributes = $entity->getChildren()->reduce( - fn (array $result, Product $child): array => $result + array_map($this->propertyFormatter, $child->getOptions()->getElements()), - array_map($this->propertyFormatter, $this->applyPropertyGroupsFilter($entity)) - ); - return $attributes ? '|' . implode('|', array_values($attributes)) . '|' : ''; + // 1. Bazowe właściwości (dziedziczone lub bezpośrednie) + $properties = $this->applyPropertyGroupsFilter($entity); + $attributes = $properties ? array_map($this->propertyFormatter, $properties) : []; + + // 2. Pobieranie opcji bez ładowania całych encji dzieci + if ($entity->getParentId() !== null) { + // Jesteśmy w wariancie - pobieramy jego konkretne opcje + $options = $entity->getOptions() ? $entity->getOptions()->getElements() : []; + $attributes = array_merge($attributes, array_map($this->propertyFormatter, $options)); + } else { + // Jesteśmy w produkcie głównym - pobieramy agregację opcji ze wszystkich wariantów + $configuratorSettings = $entity->getConfiguratorSettings(); + if ($configuratorSettings) { + $options = []; + foreach ($configuratorSettings as $setting) { + if ($setting->getOption()) { + $options[] = $setting->getOption(); + } + } + $attributes = array_merge($attributes, array_map($this->propertyFormatter, $options)); + } + } + + return $attributes ? '|' . implode('|', array_unique(array_values($attributes))) . '|' : ''; } public function getCompatibleEntityTypes(): array @@ -51,10 +68,11 @@ private function applyPropertyGroupsFilter(Product $product): array $disabledProperties = $this->exportSettings->getDisabledPropertyGroups(); if (!$disabledProperties) { - return $product->getProperties()->getElements(); + return $product->getProperties() ? $product->getProperties()->getElements() : []; } + return $product->getProperties() - ->filter(fn (PropertyGroupOptionEntity $option): bool => !in_array($option->getGroupId(), $disabledProperties)) - ->getElements(); + ->filter(fn (PropertyGroupOptionEntity $option): bool => !in_array($option->getGroupId(), $disabledProperties)) + ->getElements(); } } From d1c9c0d952aeb2bd8c4cbea199121de57152715e Mon Sep 17 00:00:00 2001 From: robertsaternus Date: Thu, 21 May 2026 16:43:08 +0200 Subject: [PATCH 2/3] Support master ID --- src/Export/Data/Entity/ProductEntity.php | 10 ++++--- .../Data/Factory/ProductEntityFactory.php | 6 +---- src/Export/ExportProducts.php | 27 +++++++++++++++---- src/Export/Field/CategoryPath.php | 2 +- src/Export/Field/FilterAttributes.php | 9 +++---- src/Export/Stream/ConsoleOutput.php | 2 +- src/Export/Stream/CsvFile.php | 2 +- src/Resources/config/services.xml | 2 +- 8 files changed, 37 insertions(+), 23 deletions(-) diff --git a/src/Export/Data/Entity/ProductEntity.php b/src/Export/Data/Entity/ProductEntity.php index ab4a3dd1..dcc92278 100644 --- a/src/Export/Data/Entity/ProductEntity.php +++ b/src/Export/Data/Entity/ProductEntity.php @@ -96,10 +96,12 @@ public function toArray(): array { $cachedProductFieldNames = array_map(fn (FieldInterface $field) => $field->getName(), iterator_to_array($this->cachedProductFields)); $fields = array_filter($this->productFields, fn (FieldInterface $productField) => !in_array($productField->getName(), $cachedProductFieldNames)); - $isVariant = $this->product->getId() !== $this->product->getParentId() && isset($this->parent); - $defaultFields = [ + $isVariant = $this->product->getParentId() !== null; + $resolvedParent = $this->parent ?? $this->product->getParent(); + + $defaultFields = [ 'ProductNumber' => $this->product->getProductNumber(), - 'Master' => $isVariant ? $this->parent->getProductNumber() : $this->product->getProductNumber(), + 'Master' => ($isVariant && $resolvedParent) ? $resolvedParent->getProductNumber() : $this->product->getProductNumber(), 'Name' => (string) $this->product->getTranslation('name'), 'FilterAttributes' => $this->getFilterAttributes(), 'CustomFields' => $this->getCustomFields(), @@ -109,7 +111,7 @@ public function toArray(): array $fields, fn (array $fields, FieldInterface $field): array => array_merge( $fields, - [$field->getName() => ($this->getAdditionalCache($field->getName()) ?? $field->getValue($isVariant ? $this->parent : $this->product))] + [$field->getName() => ($this->getAdditionalCache($field->getName()) ?? $field->getValue($isVariant && $resolvedParent ? $resolvedParent : $this->product))] ), $defaultFields ); diff --git a/src/Export/Data/Factory/ProductEntityFactory.php b/src/Export/Data/Factory/ProductEntityFactory.php index bf4cb58e..e5249677 100644 --- a/src/Export/Data/Factory/ProductEntityFactory.php +++ b/src/Export/Data/Factory/ProductEntityFactory.php @@ -23,7 +23,7 @@ public function __construct( PropertyFormatter $propertyFormatter, FieldsProvider $fieldsProviders, CurrencyFieldsProvider $currencyFieldsProvider, - \Traversable $variantFields + \Traversable $variantFields, ) { $this->propertyFormatter = $propertyFormatter; $this->fieldsProvider = $fieldsProviders; @@ -44,9 +44,6 @@ public function createEntities(Entity $entity, string $producedType = ProductEnt $fields = array_merge($this->fieldsProvider->getFields($producedType), $this->currencyFieldsProvider->getCurrencyFields()); if ($entity->getParentId() !== null) { - // To jest WARIANT. - // Tworzymy bazową encję ProductEntity (odpowiednik rodzica), ponieważ wariant w - // Shopware posiada w sobie dziedziczone dane od rodzica (nazwa, opis, producent). $pseudoParent = new $producedType($entity, new \ArrayIterator($fields), new \ArrayIterator()); yield new VariantEntity( @@ -56,7 +53,6 @@ public function createEntities(Entity $entity, string $producedType = ProductEnt iterator_to_array($this->variantFields) ); } else { - // To jest PRODUKT GŁÓWNY (Rodzic lub samodzielny produkt). yield new $producedType($entity, new \ArrayIterator($fields), new \ArrayIterator()); } } diff --git a/src/Export/ExportProducts.php b/src/Export/ExportProducts.php index c040a9e0..b21a3ad6 100644 --- a/src/Export/ExportProducts.php +++ b/src/Export/ExportProducts.php @@ -6,7 +6,6 @@ use Omikron\FactFinder\Shopware6\Export\Data\Entity\ProductEntity as ExportProductEntity; use Shopware\Core\Framework\DataAbstractionLayer\Search\Criteria; -use Shopware\Core\Framework\DataAbstractionLayer\Search\Filter\EqualsFilter; use Shopware\Core\System\SalesChannel\Entity\SalesChannelRepository; use Shopware\Core\System\SalesChannel\SalesChannelContext; @@ -35,7 +34,27 @@ public function getByContext(SalesChannelContext $context, int $batchSize = 100) break; } + $parentIds = []; + // 1. Zbieramy unikalne ID rodziców z bieżącej paczki foreach ($products->getElements() as $product) { + if ($product->getParentId() !== null) { + $parentIds[$product->getParentId()] = true; + } + } + + $parents = []; + if (!empty($parentIds)) { + $parentCriteria = new Criteria(array_keys($parentIds)); + $parentCriteria->addAssociation('categories'); + $parentCriteria->addAssociation('categoriesRo'); + $parents = $this->productRepository->search($parentCriteria, $context)->getElements(); + } + + foreach ($products->getElements() as $product) { + if ($product->getParentId() !== null && isset($parents[$product->getParentId()])) { + $product->setParent($parents[$product->getParentId()]); + } + yield $product; } @@ -71,6 +90,8 @@ private function getCriteria(int $batchSize, int $offset): Criteria $criteria = new Criteria(); $criteria->setLimit($batchSize); $criteria->setOffset($offset); + $criteria->setTotalCountMode(Criteria::TOTAL_COUNT_MODE_NONE); + $criteria->addAssociation('categories'); $criteria->addAssociation('categoriesRo'); $criteria->addAssociation('manufacturer'); @@ -87,10 +108,6 @@ private function getCriteria(int $batchSize, int $offset): Criteria $criteria->addAssociation($association); } - // UWAGA: Usunęliśmy addFilter(new EqualsFilter('parentId', null)); - // Chcemy eksportować płasko wszystko: zarówno rodziców jak i warianty, - // więc nie filtrujemy tutaj po parentId! - return $criteria; } } diff --git a/src/Export/Field/CategoryPath.php b/src/Export/Field/CategoryPath.php index 7f61cbd4..b6476736 100644 --- a/src/Export/Field/CategoryPath.php +++ b/src/Export/Field/CategoryPath.php @@ -61,6 +61,6 @@ private function getCategories(Entity $entity): CategoryCollection return new CategoryCollection([$entity]); } - return $entity->getCategories(); + return $entity->getCategories() ?? new CategoryCollection(); } } diff --git a/src/Export/Field/FilterAttributes.php b/src/Export/Field/FilterAttributes.php index d6b0cdd0..a3553cc2 100644 --- a/src/Export/Field/FilterAttributes.php +++ b/src/Export/Field/FilterAttributes.php @@ -32,25 +32,24 @@ public function getName(): string */ public function getValue(Entity $entity): string { - // 1. Bazowe właściwości (dziedziczone lub bezpośrednie) $properties = $this->applyPropertyGroupsFilter($entity); $attributes = $properties ? array_map($this->propertyFormatter, $properties) : []; - // 2. Pobieranie opcji bez ładowania całych encji dzieci if ($entity->getParentId() !== null) { - // Jesteśmy w wariancie - pobieramy jego konkretne opcje - $options = $entity->getOptions() ? $entity->getOptions()->getElements() : []; + $options = $entity->getOptions() ? $entity->getOptions()->getElements() : []; $attributes = array_merge($attributes, array_map($this->propertyFormatter, $options)); } else { - // Jesteśmy w produkcie głównym - pobieramy agregację opcji ze wszystkich wariantów $configuratorSettings = $entity->getConfiguratorSettings(); + if ($configuratorSettings) { $options = []; + foreach ($configuratorSettings as $setting) { if ($setting->getOption()) { $options[] = $setting->getOption(); } } + $attributes = array_merge($attributes, array_map($this->propertyFormatter, $options)); } } diff --git a/src/Export/Stream/ConsoleOutput.php b/src/Export/Stream/ConsoleOutput.php index 6536d05c..e9cf142a 100644 --- a/src/Export/Stream/ConsoleOutput.php +++ b/src/Export/Stream/ConsoleOutput.php @@ -23,7 +23,7 @@ public function addEntity(array $entity): void { $this->fileResource = $this->fileResource ?? new File('php://output', 'w'); ob_start(); - $this->fileResource->fputcsv($entity, $this->delimiter); + $this->fileResource->fputcsv($entity, $this->delimiter, '"', '\\'); $this->output->writeln(rtrim(ob_get_clean())); } } diff --git a/src/Export/Stream/CsvFile.php b/src/Export/Stream/CsvFile.php index fc64b218..d54cc117 100644 --- a/src/Export/Stream/CsvFile.php +++ b/src/Export/Stream/CsvFile.php @@ -23,6 +23,6 @@ public function __construct($fileResource, string $delimiter = ';') public function addEntity(array $entity): void { - fputcsv($this->fileResource, $entity, $this->delimiter); + fputcsv($this->fileResource, $entity, $this->delimiter, '"', '\\'); } } diff --git a/src/Resources/config/services.xml b/src/Resources/config/services.xml index dc677f45..93a37a8e 100644 --- a/src/Resources/config/services.xml +++ b/src/Resources/config/services.xml @@ -45,7 +45,7 @@ ProductNumber - children.media + cover.media From a894dd0ff49f20f3c02ae6e129ff087907c2b5ea Mon Sep 17 00:00:00 2001 From: robertsaternus Date: Fri, 29 May 2026 16:53:54 +0200 Subject: [PATCH 3/3] Update --- src/Command/DataExportCommand.php | 1 + src/Export/Data/Entity/ProductEntity.php | 1 + src/Export/Data/Factory/ProductEntityFactory.php | 1 + src/Export/ExportProducts.php | 9 +++++++++ src/Export/Feed.php | 1 + src/Export/Field/CategoryPath.php | 1 + src/Export/Field/FilterAttributes.php | 1 + 7 files changed, 15 insertions(+) diff --git a/src/Command/DataExportCommand.php b/src/Command/DataExportCommand.php index 4d8af710..a19eb602 100644 --- a/src/Command/DataExportCommand.php +++ b/src/Command/DataExportCommand.php @@ -231,3 +231,4 @@ private function createFile(string $exportType, string $salesChannelId) return $this->file; } } +// diff --git a/src/Export/Data/Entity/ProductEntity.php b/src/Export/Data/Entity/ProductEntity.php index dcc92278..9e2373f9 100644 --- a/src/Export/Data/Entity/ProductEntity.php +++ b/src/Export/Data/Entity/ProductEntity.php @@ -117,3 +117,4 @@ public function toArray(): array ); } } +// diff --git a/src/Export/Data/Factory/ProductEntityFactory.php b/src/Export/Data/Factory/ProductEntityFactory.php index e5249677..44a01079 100644 --- a/src/Export/Data/Factory/ProductEntityFactory.php +++ b/src/Export/Data/Factory/ProductEntityFactory.php @@ -57,3 +57,4 @@ public function createEntities(Entity $entity, string $producedType = ProductEnt } } } +// diff --git a/src/Export/ExportProducts.php b/src/Export/ExportProducts.php index b21a3ad6..0cb70697 100644 --- a/src/Export/ExportProducts.php +++ b/src/Export/ExportProducts.php @@ -108,6 +108,15 @@ private function getCriteria(int $batchSize, int $offset): Criteria $criteria->addAssociation($association); } +// $criteria->addFilter(new \Shopware\Core\Framework\DataAbstractionLayer\Search\Filter\MultiFilter( +// \Shopware\Core\Framework\DataAbstractionLayer\Search\Filter\MultiFilter::CONNECTION_OR, +// [ +// new \Shopware\Core\Framework\DataAbstractionLayer\Search\Filter\EqualsFilter('parentId', null), +// new \Shopware\Core\Framework\DataAbstractionLayer\Search\Filter\EqualsFilter('parent.active', true) +// ] +// )); + return $criteria; } } +// diff --git a/src/Export/Feed.php b/src/Export/Feed.php index 959a08b6..37fad7fa 100644 --- a/src/Export/Feed.php +++ b/src/Export/Feed.php @@ -54,3 +54,4 @@ private function prepare(array $data): array return array_map([$this->filter, 'filterValue'], $data); } } +// diff --git a/src/Export/Field/CategoryPath.php b/src/Export/Field/CategoryPath.php index b6476736..a1e24c6a 100644 --- a/src/Export/Field/CategoryPath.php +++ b/src/Export/Field/CategoryPath.php @@ -64,3 +64,4 @@ private function getCategories(Entity $entity): CategoryCollection return $entity->getCategories() ?? new CategoryCollection(); } } +// diff --git a/src/Export/Field/FilterAttributes.php b/src/Export/Field/FilterAttributes.php index a3553cc2..5799d209 100644 --- a/src/Export/Field/FilterAttributes.php +++ b/src/Export/Field/FilterAttributes.php @@ -75,3 +75,4 @@ private function applyPropertyGroupsFilter(Product $product): array ->getElements(); } } +//