Export flat records - testing

src/Command/DataExportCommand.php

@@ -154,6 +154,9 @@ public function execute(InputInterface $input, OutputInterface $output): int
             unlink(stream_get_meta_data($this->file)['uri']);
         }
 
+        $peakMemory = memory_get_peak_usage(true) / 1024 / 1024;
+        $output->writeln(sprintf('<info>Peak Memory Usage: %.2f MB</info>', $peakMemory));
+
         return 0;
     }
 
@@ -228,3 +231,4 @@ private function createFile(string $exportType, string $salesChannelId)
         return $this->file;
     }
 }
+//

src/Export/Data/Entity/ProductEntity.php

@@ -96,10 +96,12 @@ public function toArray(): array
     {
         $cachedProductFieldNames = array_map(fn (FieldInterface $field) => $field->getName(), iterator_to_array($this->cachedProductFields));
         $fields                  = array_filter($this->productFields, fn (FieldInterface $productField) => !in_array($productField->getName(), $cachedProductFieldNames));
-        $isVariant               = $this->product->getId() !== $this->product->getParentId() && isset($this->parent);
-        $defaultFields           = [
+        $isVariant = $this->product->getParentId() !== null;
+        $resolvedParent = $this->parent ?? $this->product->getParent();
+
+        $defaultFields = [
             'ProductNumber'    => $this->product->getProductNumber(),
-            'Master'           => $isVariant ? $this->parent->getProductNumber() : $this->product->getProductNumber(),
+            'Master'           => ($isVariant && $resolvedParent) ? $resolvedParent->getProductNumber() : $this->product->getProductNumber(),
             'Name'             => (string) $this->product->getTranslation('name'),
             'FilterAttributes' => $this->getFilterAttributes(),
             'CustomFields'     => $this->getCustomFields(),
@@ -109,9 +111,10 @@ public function toArray(): array
             $fields,
             fn (array $fields, FieldInterface $field): array => array_merge(
                 $fields,
-                [$field->getName() => ($this->getAdditionalCache($field->getName()) ?? $field->getValue($isVariant ? $this->parent : $this->product))]
+                [$field->getName() => ($this->getAdditionalCache($field->getName()) ?? $field->getValue($isVariant && $resolvedParent ? $resolvedParent : $this->product))]
             ),
             $defaultFields
         );
     }
 }
+//

src/Export/Data/Factory/ProductEntityFactory.php

@@ -37,22 +37,24 @@ public function handle(Entity $entity): bool
     }
 
     /**
-     * @param Entity $entity
-     * @param string $producedType
-     *
-     * @return ProductEntity[]|iterable
-     *
-     * @SuppressWarnings(PHPMD.UnusedFormalParameter)
+     * @param SalesChannelProductEntity $entity
      */
     public function createEntities(Entity $entity, string $producedType = ProductEntity::class): iterable
     {
-        // @todo use spread operator?
         $fields = array_merge($this->fieldsProvider->getFields($producedType), $this->currencyFieldsProvider->getCurrencyFields());
-        $parent = new $producedType($entity, new \ArrayIterator($fields), new \ArrayIterator());
-        if ($entity->getChildCount()) {
-            yield from $entity->getChildren()->map(fn (
-                SalesChannelProductEntity $child) => new VariantEntity($child, $parent->toArray(), $this->propertyFormatter, iterator_to_array($this->variantFields)));
+
+        if ($entity->getParentId() !== null) {
+            $pseudoParent = new $producedType($entity, new \ArrayIterator($fields), new \ArrayIterator());
+
+            yield new VariantEntity(
+                $entity,
+                $pseudoParent->toArray(),
+                $this->propertyFormatter,
+                iterator_to_array($this->variantFields)
+            );
+        } else {
+            yield new $producedType($entity, new \ArrayIterator($fields), new \ArrayIterator());
         }
-        yield $parent;
     }
 }
+//

src/Export/ExportProducts.php

@@ -6,7 +6,6 @@
 
 use Omikron\FactFinder\Shopware6\Export\Data\Entity\ProductEntity as ExportProductEntity;
 use Shopware\Core\Framework\DataAbstractionLayer\Search\Criteria;
-use Shopware\Core\Framework\DataAbstractionLayer\Search\Filter\EqualsFilter;
 use Shopware\Core\System\SalesChannel\Entity\SalesChannelRepository;
 use Shopware\Core\System\SalesChannel\SalesChannelContext;
 
@@ -25,12 +24,59 @@ public function __construct(SalesChannelRepository $productRepository, array $cu
 
     public function getByContext(SalesChannelContext $context, int $batchSize = 100): iterable
     {
-        $criteria = $this->getCriteria($batchSize);
-        $products = $this->productRepository->search($criteria, $context);
-        while ($products->count()) {
-            yield from $products;
-            $criteria->setOffset($criteria->getOffset() + $criteria->getLimit());
+        $offset = 0;
+
+        while (true) {
+            $criteria = $this->getCriteria($batchSize, $offset);
             $products = $this->productRepository->search($criteria, $context);
+
+            if ($products->count() === 0) {
+                break;
+            }
+
+            $parentIds = [];
+            // 1. Zbieramy unikalne ID rodziców z bieżącej paczki
+            foreach ($products->getElements() as $product) {
+                if ($product->getParentId() !== null) {
+                    $parentIds[$product->getParentId()] = true;
+                }
+            }
+
+            $parents = [];
+            if (!empty($parentIds)) {
+                $parentCriteria = new Criteria(array_keys($parentIds));
+                $parentCriteria->addAssociation('categories');
+                $parentCriteria->addAssociation('categoriesRo');
+                $parents = $this->productRepository->search($parentCriteria, $context)->getElements();
+            }
+
+            foreach ($products->getElements() as $product) {
+                if ($product->getParentId() !== null && isset($parents[$product->getParentId()])) {
+                    $product->setParent($parents[$product->getParentId()]);
+                }
+
+                yield $product;
+            }
+
+            $products->clear();
+            unset($products, $criteria);
+            gc_collect_cycles();
+
+            // --- DEBUG PAMIĘCI START ---
+            // Przeliczamy bajty na megabajty dla czytelności
+            $memoryUsageMB = memory_get_usage(true) / 1024 / 1024;
+            $peakMemoryMB  = memory_get_peak_usage(true) / 1024 / 1024;
+
+            echo sprintf(
+                "[%s] Offset: %d | Memory: %.2f MB | Peak: %.2f MB\n",
+                date('H:i:s'),
+                $offset,
+                $memoryUsageMB,
+                $peakMemoryMB
+            );
+            // --- DEBUG PAMIĘCI END ---
+
+            $offset += $batchSize;
         }
     }
 
@@ -39,25 +85,38 @@ public function getProducedExportEntityType(): string
         return ExportProductEntity::class;
     }
 
-    private function getCriteria(int $batchSize): Criteria
+    private function getCriteria(int $batchSize, int $offset): Criteria
     {
         $criteria = new Criteria();
         $criteria->setLimit($batchSize);
+        $criteria->setOffset($offset);
+        $criteria->setTotalCountMode(Criteria::TOTAL_COUNT_MODE_NONE);
+
         $criteria->addAssociation('categories');
         $criteria->addAssociation('categoriesRo');
-        $criteria->addAssociation('children.options.group');
         $criteria->addAssociation('manufacturer');
-        $criteria->addAssociation('properties');
-        $criteria->addAssociation('customFields');
         $criteria->addAssociation('properties.group');
-        $criteria->addAssociation('seoUrls');
+        $criteria->addAssociation('options.group');
         $criteria->addAssociation('media');
-        $criteria->addAssociation('children.cover.media');
+        $criteria->addAssociation('cover.media');
+        $criteria->addAssociation('seoUrls');
+        $criteria->addAssociation('customFields');
+
+        $criteria->addAssociation('configuratorSettings.option.group');
+
         foreach ($this->customAssociations as $association) {
             $criteria->addAssociation($association);
         }
-        $criteria->addFilter(new EqualsFilter('parentId', null));
+
+//        $criteria->addFilter(new \Shopware\Core\Framework\DataAbstractionLayer\Search\Filter\MultiFilter(
+//            \Shopware\Core\Framework\DataAbstractionLayer\Search\Filter\MultiFilter::CONNECTION_OR,
+//            [
+//                new \Shopware\Core\Framework\DataAbstractionLayer\Search\Filter\EqualsFilter('parentId', null),
+//                new \Shopware\Core\Framework\DataAbstractionLayer\Search\Filter\EqualsFilter('parent.active', true)
+//            ]
+//        ));
 
         return $criteria;
     }
 }
+//

src/Export/Feed.php

@@ -29,9 +29,16 @@ public function generate(StreamInterface $stream, array $columns): void
         $stream->addEntity($columns);
         $emptyRecord = array_combine($columns, array_fill(0, count($columns), ''));
 
+        $i = 0;
+
         foreach ($this->getEntities() as $entity) {
             $entityData = array_merge($emptyRecord, array_intersect_key($entity->toArray(), $emptyRecord));
             $stream->addEntity($this->prepare($entityData));
+
+            unset($entity, $entityData);
+            if (++$i % 500 === 0) {
+                gc_collect_cycles();
+            }
         }
     }
 
@@ -47,3 +54,4 @@ private function prepare(array $data): array
         return array_map([$this->filter, 'filterValue'], $data);
     }
 }
+//

src/Export/Field/CategoryPath.php

@@ -61,6 +61,7 @@ private function getCategories(Entity $entity): CategoryCollection
             return new CategoryCollection([$entity]);
         }
 
-        return $entity->getCategories();
+        return $entity->getCategories() ?? new CategoryCollection();
     }
 }
+//

src/Export/Field/FilterAttributes.php

@@ -29,16 +29,32 @@ public function getName(): string
 
     /**
      * @param Product $entity
-     *
-     * @return string
      */
     public function getValue(Entity $entity): string
     {
-        $attributes = $entity->getChildren()->reduce(
-            fn (array $result, Product $child): array => $result + array_map($this->propertyFormatter, $child->getOptions()->getElements()),
-            array_map($this->propertyFormatter, $this->applyPropertyGroupsFilter($entity))
-        );
-        return $attributes ? '|' . implode('|', array_values($attributes)) . '|' : '';
+        $properties = $this->applyPropertyGroupsFilter($entity);
+        $attributes = $properties ? array_map($this->propertyFormatter, $properties) : [];
+
+        if ($entity->getParentId() !== null) {
+            $options    = $entity->getOptions() ? $entity->getOptions()->getElements() : [];
+            $attributes = array_merge($attributes, array_map($this->propertyFormatter, $options));
+        } else {
+            $configuratorSettings = $entity->getConfiguratorSettings();
+
+            if ($configuratorSettings) {
+                $options = [];
+
+                foreach ($configuratorSettings as $setting) {
+                    if ($setting->getOption()) {
+                        $options[] = $setting->getOption();
+                    }
+                }
+
+                $attributes = array_merge($attributes, array_map($this->propertyFormatter, $options));
+            }
+        }
+
+        return $attributes ? '|' . implode('|', array_unique(array_values($attributes))) . '|' : '';
     }
 
     public function getCompatibleEntityTypes(): array
@@ -51,10 +67,12 @@ private function applyPropertyGroupsFilter(Product $product): array
         $disabledProperties = $this->exportSettings->getDisabledPropertyGroups();
 
         if (!$disabledProperties) {
-            return $product->getProperties()->getElements();
+            return $product->getProperties() ? $product->getProperties()->getElements() : [];
         }
+
         return $product->getProperties()
-                       ->filter(fn (PropertyGroupOptionEntity $option): bool => !in_array($option->getGroupId(), $disabledProperties))
-                       ->getElements();
+            ->filter(fn (PropertyGroupOptionEntity $option): bool => !in_array($option->getGroupId(), $disabledProperties))
+            ->getElements();
     }
 }
+//

src/Export/Stream/ConsoleOutput.php

@@ -23,7 +23,7 @@ public function addEntity(array $entity): void
     {
         $this->fileResource = $this->fileResource ?? new File('php://output', 'w');
         ob_start();
-        $this->fileResource->fputcsv($entity, $this->delimiter);
+        $this->fileResource->fputcsv($entity, $this->delimiter, '"', '\\');
         $this->output->writeln(rtrim(ob_get_clean()));
     }
 }

src/Export/Stream/CsvFile.php

@@ -23,6 +23,6 @@ public function __construct($fileResource, string $delimiter = ';')
 
     public function addEntity(array $entity): void
     {
-        fputcsv($this->fileResource, $entity, $this->delimiter);
+        fputcsv($this->fileResource, $entity, $this->delimiter, '"', '\\');
     }
 }

src/Resources/config/services.xml

@@ -45,7 +45,7 @@
             <parameter key="productNumber">ProductNumber</parameter>
         </parameter>
         <parameter key="factfinder.export.associations" type="collection">
-            <parameter key="variant_cover">children.media</parameter>
+            <parameter key="variant_cover">cover.media</parameter>
         </parameter>
         <parameter key="factfinder.category_page.add_params" type="collection">
         </parameter>