Started to work on an import possibility for Partkeepr databases

composer.json

@@ -9,6 +9,7 @@
         "ext-intl": "*",
         "ext-json": "*",
         "ext-mbstring": "*",
+        "ext-dom": "*",
         "beberlei/doctrineextensions": "^1.2",
         "brick/math": "^0.8.15",
         "composer/package-versions-deprecated": "1.11.99.4",

src/Command/Migrations/ImportPartKeeprCommand.php

@@ -0,0 +1,96 @@
+<?php
+/*
+ * This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
+ *
+ *  Copyright (C) 2019 - 2023 Jan Böhmer (https://github.com/jbtronics)
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU Affero General Public License as published
+ *  by the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Affero General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Affero General Public License
+ *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+namespace App\Command\Migrations;
+
+use App\Services\ImportExportSystem\PartkeeprImporter;
+use App\Services\Misc\MySQLDumpXMLConverter;
+use Doctrine\ORM\EntityManagerInterface;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Console\Style\SymfonyStyle;
+
+class ImportPartKeeprCommand extends Command
+{
+
+    protected static $defaultName = 'partdb:import-partkeepr';
+
+    protected EntityManagerInterface $em;
+    protected PartkeeprImporter $importer;
+    protected MySQLDumpXMLConverter $xml_converter;
+
+    public function __construct(EntityManagerInterface $em, PartkeeprImporter $importer, MySQLDumpXMLConverter $xml_converter)
+    {
+        parent::__construct(self::$defaultName);
+        $this->em = $em;
+        $this->importer = $importer;
+        $this->xml_converter = $xml_converter;
+    }
+
+    protected function configure()
+    {
+        $this->setDescription('Import a PartKeepr database dump into Part-DB');
+
+        $this->addArgument('file', InputArgument::REQUIRED, 'The file to which should be imported.');
+    }
+
+    public function execute(InputInterface $input, OutputInterface $output)
+    {
+        $io = new SymfonyStyle($input, $output);
+
+        $input_path = $input->getArgument('file');
+
+        //Make more checks here
+        //$io->confirm('This will delete all data in the database. Do you want to continue?', false);
+
+        //Purge the databse, so we will not have any conflicts
+        $this->importer->purgeDatabaseForImport();
+
+        //Convert the XML file to an array
+        $xml = file_get_contents($input_path);
+        $data = $this->xml_converter->convertMySQLDumpXMLDataToArrayStructure($xml);
+
+        //Import the data
+        $this->doImport($io, $data);
+
+        return 0;
+    }
+
+    private function doImport(SymfonyStyle $io, array $data)
+    {
+        //First import the distributors
+        $io->info('Importing distributors...');
+        $count = $this->importer->importDistributors($data);
+        $io->success('Imported '.$count.' distributors.');
+
+        //Import the measurement units
+        $io->info('Importing part measurement units...');
+        $count = $this->importer->importPartUnits($data);
+        $io->success('Imported '.$count.' measurement units.');
+
+        //Import manufacturers
+        $io->info('Importing manufacturers...');
+        $count = $this->importer->importManufacturers($data);
+        $io->success('Imported '.$count.' manufacturers.');
+    }
+
+}

src/Doctrine/Purger/ResetAutoIncrementORMPurger.php

@@ -179,7 +179,7 @@ class ResetAutoIncrementORMPurger implements PurgerInterface, ORMPurgerInterface
             }
 
             // If the table is excluded, skip it as well
-            if (array_search($tbl, $this->excluded) !== false) {
+            if (in_array($tbl, $this->excluded, true)) {
                 continue;
             }
 

src/Services/ImportExportSystem/PartkeeprImporter.php

@@ -0,0 +1,154 @@
+<?php
+/*
+ * This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
+ *
+ *  Copyright (C) 2019 - 2023 Jan Böhmer (https://github.com/jbtronics)
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU Affero General Public License as published
+ *  by the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Affero General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Affero General Public License
+ *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+namespace App\Services\ImportExportSystem;
+
+use App\Doctrine\Purger\ResetAutoIncrementORMPurger;
+use App\Doctrine\Purger\ResetAutoIncrementPurgerFactory;
+use App\Entity\Base\AbstractDBElement;
+use App\Entity\Parts\Manufacturer;
+use App\Entity\Parts\MeasurementUnit;
+use App\Entity\Parts\Part;
+use App\Entity\Parts\Supplier;
+use Doctrine\Bundle\FixturesBundle\Purger\ORMPurgerFactory;
+use Doctrine\Bundle\FixturesBundle\Purger\PurgerFactory;
+use Doctrine\ORM\EntityManagerInterface;
+use Doctrine\ORM\Mapping\ClassMetadataInfo;
+
+class PartkeeprImporter
+{
+
+    protected EntityManagerInterface $em;
+
+    public function __construct(EntityManagerInterface $em)
+    {
+        $this->em = $em;
+    }
+
+    public function purgeDatabaseForImport(): void
+    {
+        //Versions with "" are needed !!
+        $purger = new ResetAutoIncrementORMPurger($this->em, ['users', '"users"', 'groups', '"groups"', 'u2f_keys', 'internal', 'migration_versions']);
+        $purger->purge();
+    }
+
+    /**
+     * Imports the distributors from the given data.
+     * @param array $data The data to import (associated array, containing a 'distributor' key
+     * @return int The number of imported distributors
+     */
+    public function importDistributors(array $data): int
+    {
+        if (!isset($data['distributor'])) {
+            throw new \RuntimeException('$data must contain a "distributor" key!');
+        }
+
+        $distributor_data = $data['distributor'];
+
+        foreach ($distributor_data as $distributor) {
+            $supplier = new Supplier();
+            $supplier->setName($distributor['name']);
+            $supplier->setWebsite($distributor['url'] ?? '');
+            $supplier->setAddress($distributor['address'] ?? '');
+            $supplier->setPhoneNumber($distributor['phone'] ?? '');
+            $supplier->setFaxNumber($distributor['fax'] ?? '');
+            $supplier->setEmailAddress($distributor['email'] ?? '');
+            $supplier->setComment($distributor['comment']);
+            $supplier->setAutoProductUrl($distributor['skuurl'] ?? '');
+
+            $this->setIDOfEntity($supplier, $distributor['id']);
+            $this->em->persist($supplier);
+        }
+
+        $this->em->flush();
+
+        return count($distributor_data);
+    }
+
+    public function importManufacturers(array $data): int
+    {
+        if (!isset($data['manufacturer'])) {
+            throw new \RuntimeException('$data must contain a "manufacturer" key!');
+        }
+
+        $manufacturer_data = $data['manufacturer'];
+
+        $max_id = 0;
+
+        //Assign a parent manufacturer to all manufacturers, as partkeepr has a lot of manufacturers by default
+        $parent_manufacturer = new Manufacturer();
+        $parent_manufacturer->setName('PartKeepr');
+        $parent_manufacturer->setNotSelectable(true);
+
+        foreach ($manufacturer_data as $manufacturer) {
+            $entity = new Manufacturer();
+            $entity->setName($manufacturer['name']);
+            $entity->setWebsite($manufacturer['url'] ?? '');
+            $entity->setAddress($manufacturer['address'] ?? '');
+            $entity->setPhoneNumber($manufacturer['phone'] ?? '');
+            $entity->setFaxNumber($manufacturer['fax'] ?? '');
+            $entity->setEmailAddress($manufacturer['email'] ?? '');
+            $entity->setComment($manufacturer['comment']);
+            $entity->setParent($parent_manufacturer);
+
+            $this->setIDOfEntity($entity, $manufacturer['id']);
+            $this->em->persist($entity);
+
+            $max_id = max($max_id, $manufacturer['id']);
+        }
+
+        //Set the ID of the parent manufacturer to the max ID + 1, to avoid trouble with the auto increment
+        $this->setIDOfEntity($parent_manufacturer, $max_id + 1);
+        $this->em->persist($parent_manufacturer);
+
+        $this->em->flush();
+
+        return count($manufacturer_data);
+    }
+
+    public function importPartUnits(array $data): int
+    {
+        if (!isset($data['partunit'])) {
+            throw new \RuntimeException('$data must contain a "partunit" key!');
+        }
+
+        $partunit_data = $data['partunit'];
+        foreach ($partunit_data as $partunit) {
+            $unit = new MeasurementUnit();
+            $unit->setName($partunit['name']);
+            $unit->setUnit($partunit['shortName'] ?? null);
+
+            $this->setIDOfEntity($unit, $partunit['id']);
+            $this->em->persist($unit);
+        }
+
+        $this->em->flush();
+
+        return count($partunit_data);
+    }
+
+    public function setIDOfEntity(AbstractDBElement $element, int $id): void
+    {
+        $metadata = $this->em->getClassMetadata(get_class($element));
+        $metadata->setIdGeneratorType(ClassMetadataInfo::GENERATOR_TYPE_NONE);
+        $metadata->setIdGenerator(new \Doctrine\ORM\Id\AssignedGenerator());
+        $metadata->setIdentifierValues($element, ['id' => $id]);
+    }
+}

src/Services/Misc/MySQLDumpXMLConverter.php

@@ -0,0 +1,107 @@
+<?php
+/*
+ * This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
+ *
+ *  Copyright (C) 2019 - 2023 Jan Böhmer (https://github.com/jbtronics)
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU Affero General Public License as published
+ *  by the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Affero General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Affero General Public License
+ *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+namespace App\Services\Misc;
+
+class MySQLDumpXMLConverter
+{
+
+    /**
+     * Converts a MySQL dump XML file to an associative array structure in the following form
+     * [
+     *    'table_name' => [
+     *       [
+     *         'column_name' => 'value',
+     *        'column_name' => 'value',
+     *       ...
+     *      ],
+     *     [
+     *        'column_name' => 'value',
+     *       'column_name' => 'value',
+     *     ...
+     *    ],
+     *   ...
+     * ],
+     *
+     * @param  string  $xml_string The XML string to convert
+     * @return array The associative array structure
+     */
+    public function convertMySQLDumpXMLDataToArrayStructure(string $xml_string): array
+    {
+        $dom = new \DOMDocument();
+        $dom->loadXML($xml_string);
+
+        //Check that the root node is a <mysqldump> node
+        $root = $dom->documentElement;
+        if ($root->nodeName !== 'mysqldump') {
+            throw new \InvalidArgumentException('The given XML string is not a valid MySQL dump XML file!');
+        }
+
+        //Get all <database> nodes (there must be exactly one)
+        $databases = $root->getElementsByTagName('database');
+        if ($databases->length !== 1) {
+            throw new \InvalidArgumentException('The given XML string is not a valid MySQL dump XML file!');
+        }
+
+        //Get the <database> node
+        $database = $databases->item(0);
+
+        //Get all <table_data> nodes
+        $tables = $database->getElementsByTagName('table_data');
+        $table_data = [];
+
+        //Iterate over all <table> nodes and convert them to arrays
+        foreach ($tables as $table) {
+            $table_data[$table->getAttribute('name')] = $this->convertTableToArray($table);
+        }
+
+        return $table_data;
+    }
+
+    private function convertTableToArray(\DOMElement $table): array
+    {
+        $table_data = [];
+
+        //Get all <row> nodes
+        $rows = $table->getElementsByTagName('row');
+
+        //Iterate over all <row> nodes and convert them to arrays
+        foreach ($rows as $row) {
+            $table_data[] = $this->convertTableRowToArray($row);
+        }
+
+        return $table_data;
+    }
+
+    private function convertTableRowToArray(\DOMElement $table_row): array
+    {
+        $row_data = [];
+
+        //Get all <field> nodes
+        $fields = $table_row->getElementsByTagName('field');
+
+        //Iterate over all <field> nodes
+        foreach ($fields as $field) {
+            $row_data[$field->getAttribute('name')] = $field->nodeValue;
+        }
+
+        return $row_data;
+    }
+}

tests/Services/Misc/MySQLDumpXMLConverterTest.php

@@ -0,0 +1,54 @@
+<?php
+/*
+ * This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
+ *
+ *  Copyright (C) 2019 - 2023 Jan Böhmer (https://github.com/jbtronics)
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU Affero General Public License as published
+ *  by the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Affero General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Affero General Public License
+ *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+namespace App\Tests\Services\Misc;
+
+use App\Services\Misc\MySQLDumpXMLConverter;
+use PHPUnit\Framework\TestCase;
+
+class MySQLDumpXMLConverterTest extends TestCase
+{
+
+    public function testConvertMySQLDumpXMLDataToArrayStructure()
+    {
+        $service = new MySQLDumpXMLConverter();
+
+        //Load the test XML file
+        $xml_string = file_get_contents(__DIR__.'/../../assets/partkeepr_import_test.xml');
+
+        $result = $service->convertMySQLDumpXMLDataToArrayStructure($xml_string);
+
+        //Check that the result is an array
+        $this->assertIsArray($result);
+
+        //Must contain 36 tables
+        $this->assertCount(50, $result);
+
+        //Must have a table called "footprints"
+        $this->assertArrayHasKey('footprint', $result);
+
+        //Must have 36 entry in the "footprints" table
+        $this->assertCount(36, $result['footprint']);
+
+        $this->assertSame('1', $result['footprint'][0]['id']);
+        $this->assertSame('CBGA-32', $result['footprint'][0]['name']);
+
+    }
+}