Merge branch 'part_import'

This commit is contained in:
Jan Böhmer 2023-03-17 00:11:16 +01:00
commit 14740fad58
13 changed files with 583 additions and 8 deletions

View file

@ -24,6 +24,7 @@
"gregwar/captcha-bundle": "^2.1.0",
"hslavich/oneloginsaml-bundle": "^2.10",
"jbtronics/2fa-webauthn": "^1.0.0",
"league/csv": "^9.8.0",
"league/html-to-markdown": "^5.0.1",
"liip/imagine-bundle": "^2.2",
"nelexa/zip": "^4.0",

86
composer.lock generated
View file

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "73b35aff40231c2fe1ebf72c1d098689",
"content-hash": "43882c51b2efa31f08c345a94661916c",
"packages": [
{
"name": "beberlei/assert",
@ -2746,6 +2746,90 @@
],
"time": "2023-01-02T13:28:00+00:00"
},
{
"name": "league/csv",
"version": "9.8.0",
"source": {
"type": "git",
"url": "https://github.com/thephpleague/csv.git",
"reference": "9d2e0265c5d90f5dd601bc65ff717e05cec19b47"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/thephpleague/csv/zipball/9d2e0265c5d90f5dd601bc65ff717e05cec19b47",
"reference": "9d2e0265c5d90f5dd601bc65ff717e05cec19b47",
"shasum": ""
},
"require": {
"ext-json": "*",
"ext-mbstring": "*",
"php": "^7.4 || ^8.0"
},
"require-dev": {
"ext-curl": "*",
"ext-dom": "*",
"friendsofphp/php-cs-fixer": "^v3.4.0",
"phpstan/phpstan": "^1.3.0",
"phpstan/phpstan-phpunit": "^1.0.0",
"phpstan/phpstan-strict-rules": "^1.1.0",
"phpunit/phpunit": "^9.5.11"
},
"suggest": {
"ext-dom": "Required to use the XMLConverter and or the HTMLConverter classes",
"ext-iconv": "Needed to ease transcoding CSV using iconv stream filters"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "9.x-dev"
}
},
"autoload": {
"files": [
"src/functions_include.php"
],
"psr-4": {
"League\\Csv\\": "src"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Ignace Nyamagana Butera",
"email": "nyamsprod@gmail.com",
"homepage": "https://github.com/nyamsprod/",
"role": "Developer"
}
],
"description": "CSV data manipulation made easy in PHP",
"homepage": "https://csv.thephpleague.com",
"keywords": [
"convert",
"csv",
"export",
"filter",
"import",
"read",
"transform",
"write"
],
"support": {
"docs": "https://csv.thephpleague.com",
"issues": "https://github.com/thephpleague/csv/issues",
"rss": "https://github.com/thephpleague/csv/releases.atom",
"source": "https://github.com/thephpleague/csv"
},
"funding": [
{
"url": "https://github.com/sponsors/nyamsprod",
"type": "github"
}
],
"time": "2022-01-04T00:13:07+00:00"
},
{
"name": "league/html-to-markdown",
"version": "5.1.0",

29
docs/usage/bom_import.md Normal file
View file

@ -0,0 +1,29 @@
---
layout: default
title: Import Bill of Material (BOM) for Projects
nav_order: 5
parent: Usage
---
# Import Bill of Material (BOM) for Projects
Part-DB supports the import of Bill of Material (BOM) files for projects. This allows you to directly import a BOM file from your ECAD software into your Part-DB project.
The import process is currently semi-automatic. This means Part-DB will take the BOM file and create entries for all parts in the BOM file in your project and assign fields like
mountnames (e.g. 'C1, C2, C3'), quantity and more.
However, you still have to assign the parts from Part-DB database to the entries (if applicable) after the import by hand,
as Part-DB can not know which part you had in mind when you designed your schematic.
## Usage
In the project view or edit click on the "Import BOM" button, below the BOM table. This will open a dialog where you can
select the BOM file you want to import and some options for the import process:
* **Type**: The format/type of the BOM file. See below for explanations of the different types.
* **Clear existing BOM entries before import**: If this is checked, all existing BOM entries, which are currently associated with the project, will be deleted before the import.
### Supported BOM file formats
* **KiCAD Pcbnew BOM (CSV file)**: A CSV file of the Bill of Material (BOM) generated by [KiCAD Pcbnew](https://www.kicad.org/).
Please note that you have to export the BOM from the PCB editor, the BOM generated by the schematic editor (Eeschema) has a different format and does not work with this type.
You can generate this BOM file by going to "File" -> "Fabrication Outputs" -> "Bill of Materials" in Pcbnew and save the file to your desired location.

View file

@ -28,17 +28,26 @@ use App\Form\ProjectSystem\ProjectBOMEntryCollectionType;
use App\Form\ProjectSystem\ProjectBuildType;
use App\Form\Type\StructuralEntityType;
use App\Helpers\Projects\ProjectBuildRequest;
use App\Services\ImportExportSystem\BOMImporter;
use App\Services\ProjectSystem\ProjectBuildHelper;
use Doctrine\Common\Collections\ArrayCollection;
use Doctrine\ORM\EntityManagerInterface;
use League\Csv\Exception;
use League\Csv\SyntaxError;
use Omines\DataTablesBundle\DataTableFactory;
use Sensio\Bundle\FrameworkExtraBundle\Configuration\ParamConverter;
use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
use Symfony\Component\Form\Extension\Core\Type\CheckboxType;
use Symfony\Component\Form\Extension\Core\Type\ChoiceType;
use Symfony\Component\Form\Extension\Core\Type\FileType;
use Symfony\Component\Form\Extension\Core\Type\SubmitType;
use Symfony\Component\HttpFoundation\Request;
use Symfony\Component\HttpFoundation\Response;
use Symfony\Component\Routing\Annotation\Route;
use Symfony\Component\Validator\Constraints\NotNull;
use Symfony\Component\Validator\Validator\ValidatorInterface;
use function Symfony\Component\Translation\t;
/**
* @Route("/project")
@ -119,6 +128,82 @@ class ProjectController extends AbstractController
]);
}
/**
* @Route("/{id}/import_bom", name="project_import_bom", requirements={"id"="\d+"})
*/
public function importBOM(Request $request, EntityManagerInterface $entityManager, Project $project,
BOMImporter $BOMImporter, ValidatorInterface $validator): Response
{
$this->denyAccessUnlessGranted('edit', $project);
$builder = $this->createFormBuilder();
$builder->add('file', FileType::class, [
'label' => 'import.file',
'required' => true,
'attr' => [
'accept' => '.csv'
]
]);
$builder->add('type', ChoiceType::class, [
'label' => 'project.bom_import.type',
'required' => true,
'choices' => [
'project.bom_import.type.kicad_pcbnew' => 'kicad_pcbnew',
]
]);
$builder->add('clear_existing_bom', CheckboxType::class, [
'label' => 'project.bom_import.clear_existing_bom',
'required' => false,
'data' => false,
'help' => 'project.bom_import.clear_existing_bom.help',
]);
$builder->add('submit', SubmitType::class, [
'label' => 'import.btn',
]);
$form = $builder->getForm();
$form->handleRequest($request);
if ($form->isSubmitted() && $form->isValid()) {
//Clear existing BOM entries if requested
if ($form->get('clear_existing_bom')->getData()) {
$project->getBomEntries()->clear();
$entityManager->flush();
}
try {
$entries = $BOMImporter->importFileIntoProject($form->get('file')->getData(), $project, [
'type' => $form->get('type')->getData(),
]);
//Validate the project entries
$errors = $validator->validateProperty($project, 'bom_entries');
//If no validation errors occured, save the changes and redirect to edit page
if (count ($errors) === 0) {
$this->addFlash('success', t('project.bom_import.flash.success', ['%count%' => count($entries)]));
$entityManager->flush();
return $this->redirectToRoute('project_edit', ['id' => $project->getID()]);
}
if (count ($errors) > 0) {
$this->addFlash('error', t('project.bom_import.flash.invalid_entries'));
}
} catch (\UnexpectedValueException $e) {
$this->addFlash('error', t('project.bom_import.flash.invalid_file', ['%message%' => $e->getMessage()]));
} catch (SyntaxError $e) {
$this->addFlash('error', t('project.bom_import.flash.invalid_file', ['%message%' => $e->getMessage()]));
}
}
return $this->renderForm('projects/import_bom.html.twig', [
'project' => $project,
'form' => $form,
'errors' => $errors ?? null,
]);
}
/**
* @Route("/add_parts", name="project_add_parts_no_id")
* @Route("/{id}/add_parts", name="project_add_parts", requirements={"id"="\d+"})

View file

@ -110,7 +110,8 @@ class ProjectBOMEntry extends AbstractDBElement
public function __construct()
{
$this->price = BigDecimal::zero()->toScale(5);
//$this->price = BigDecimal::zero()->toScale(5);
$this->price = null;
}
/**

View file

@ -0,0 +1,146 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2023 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
namespace App\Services\ImportExportSystem;
use App\Entity\ProjectSystem\Project;
use App\Entity\ProjectSystem\ProjectBOMEntry;
use InvalidArgumentException;
use League\Csv\Reader;
use Symfony\Component\HttpFoundation\File\File;
use Symfony\Component\OptionsResolver\OptionsResolver;
use Symfony\Component\Serializer\SerializerInterface;
use Symfony\Component\Validator\Validator\ValidatorInterface;
class BOMImporter
{
private const MAP_KICAD_PCB_FIELDS = [
'ID' => 'Id',
'Bezeichner' => 'Designator',
'Footprint' => 'Package',
'Stückzahl' => 'Quantity',
'Bezeichnung' => 'Designation',
'Anbieter und Referenz' => 'Supplier and ref',
];
public function __construct()
{
}
protected function configureOptions(OptionsResolver $resolver): OptionsResolver
{
$resolver->setRequired('type');
$resolver->setAllowedValues('type', ['kicad_pcbnew']);
return $resolver;
}
/**
* Converts the given file into an array of BOM entries using the given options and save them into the given project.
* The changes are not saved into the database yet.
* @param File $file
* @param array $options
* @param Project $project
* @return ProjectBOMEntry[]
*/
public function importFileIntoProject(File $file, Project $project, array $options): array
{
$bom_entries = $this->fileToBOMEntries($file, $options);
//Assign the bom_entries to the project
foreach ($bom_entries as $bom_entry) {
$project->addBomEntry($bom_entry);
}
return $bom_entries;
}
/**
* Converts the given file into an array of BOM entries using the given options.
* @param File $file
* @param array $options
* @return ProjectBOMEntry[]
*/
public function fileToBOMEntries(File $file, array $options): array
{
return $this->stringToBOMEntries($file->getContent(), $options);
}
/**
* Import string data into an array of BOM entries, which are not yet assigned to a project.
* @param string $data The data to import
* @param array $options An array of options
* @return ProjectBOMEntry[] An array of imported entries
*/
public function stringToBOMEntries(string $data, array $options): array
{
$resolver = new OptionsResolver();
$resolver = $this->configureOptions($resolver);
$options = $resolver->resolve($options);
switch ($options['type']) {
case 'kicad_pcbnew':
return $this->parseKiCADPCB($data, $options);
default:
throw new InvalidArgumentException('Invalid import type!');
}
}
private function parseKiCADPCB(string $data, array $options = []): array
{
$csv = Reader::createFromString($data);
$csv->setDelimiter(';');
$csv->setHeaderOffset(0);
$bom_entries = [];
foreach ($csv->getRecords() as $offset => $entry) {
//Translate the german field names to english
$entry = array_combine(array_map(function ($key) {
return self::MAP_KICAD_PCB_FIELDS[$key] ?? $key;
}, array_keys($entry)), $entry);
//Ensure that the entry has all required fields
if (!isset ($entry['Designator'])) {
throw new \UnexpectedValueException('Designator missing at line '.($offset + 1).'!');
}
if (!isset ($entry['Package'])) {
throw new \UnexpectedValueException('Package missing at line '.($offset + 1).'!');
}
if (!isset ($entry['Designation'])) {
throw new \UnexpectedValueException('Designation missing at line '.($offset + 1).'!');
}
if (!isset ($entry['Quantity'])) {
throw new \UnexpectedValueException('Quantity missing at line '.($offset + 1).'!');
}
$bom_entry = new ProjectBOMEntry();
$bom_entry->setName($entry['Designation'] . ' (' . $entry['Package'] . ')');
$bom_entry->setMountnames($entry['Designator'] ?? '');
$bom_entry->setComment($entry['Supplier and ref'] ?? '');
$bom_entry->setQuantity((float) ($entry['Quantity'] ?? 1));
$bom_entries[] = $bom_entry;
}
return $bom_entries;
}
}

View file

@ -51,5 +51,12 @@
{% form_theme form.bom_entries with ['form/collection_types_layout.html.twig'] %}
{{ form_errors(form.bom_entries) }}
{{ form_widget(form.bom_entries) }}
{% if entity.id %}
<a href="{{ path('project_import_bom', {'id': entity.id}) }}" class="btn btn-secondary mb-2"
{% if not is_granted('edit', entity) %}disabled="disabled"{% endif %}>
<i class="fa-solid fa-file-import fa-fw"></i>
{% trans %}project.edit.bom.import_bom{% endtrans %}
</a>
{% endif %}
</div>
{% endblock %}

View file

@ -0,0 +1,31 @@
{% extends "main_card.html.twig" %}
{% block title %}{% trans %}project.import_bom{% endtrans %}{% endblock %}
{% block before_card %}
{% if errors %}
<div class="alert alert-danger">
<h4><i class="fa-solid fa-exclamation-triangle fa-fw"></i> {% trans %}parts.import.errors.title{% endtrans %}</h4>
<ul>
{% for violation in errors %}
<li>
<b>{{ violation.propertyPath }}: </b>
{{ violation.message|trans(violation.parameters, 'validators') }}
</li>
{% endfor %}
</ul>
</div>
{% endif %}
{% endblock %}
{% block card_title %}
<i class="fa-solid fa-file-import fa-fw"></i>
{% trans %}project.import_bom{% endtrans %}{% if project %}: <i>{{ project.name }}</i>{% endif %}
{% endblock %}
{% block card_content %}
{{ form(form) }}
{% endblock %}

View file

@ -4,8 +4,21 @@
{{ datatables.datatable(datatable, 'elements/datatables/datatables', 'projects') }}
<a class="btn btn-success" {% if not is_granted('@projects.edit') %}disabled{% endif %}
href="{{ path('project_add_parts', {"id": project.id, "_redirect": app.request.requestUri}) }}">
<i class="fa-solid fa-square-plus fa-fw"></i>
{% trans %}project.info.bom_add_parts{% endtrans %}
</a>
<div class="btn-group">
<a class="btn btn-success" {% if not is_granted('@projects.edit') %}disabled{% endif %}
href="{{ path('project_add_parts', {"id": project.id, "_redirect": app.request.requestUri}) }}">
<i class="fa-solid fa-square-plus fa-fw"></i>
{% trans %}project.info.bom_add_parts{% endtrans %}
</a>
<button type="button" class="btn btn-success dropdown-toggle dropdown-toggle-split" data-bs-toggle="dropdown" aria-expanded="false">
<span class="visually-hidden">Toggle Dropdown</span>
</button>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="{{ path('project_import_bom', {'id': project.id}) }}" {% if not is_granted('edit', project) %}disabled="disabled"{% endif %}>
<i class="fa-solid fa-file-import fa-fw"></i>
{% trans %}project.edit.bom.import_bom{% endtrans %}
</a>
</li>
</ul>
</div>

View file

@ -137,5 +137,6 @@ class ApplicationAvailabilityFunctionalTest extends WebTestCase
yield ['/project/1/add_parts'];
yield ['/project/1/add_parts?parts=1,2'];
yield ['/project/1/build?n=1'];
yield ['/project/1/import_bom'];
}
}

View file

@ -0,0 +1,123 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2023 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
namespace App\Tests\Services\ImportExportSystem;
use App\Entity\ProjectSystem\Project;
use App\Entity\ProjectSystem\ProjectBOMEntry;
use App\Services\ImportExportSystem\BOMImporter;
use PHPUnit\Framework\TestCase;
use Symfony\Bundle\FrameworkBundle\Test\WebTestCase;
use Symfony\Component\HttpFoundation\File\File;
class BOMImporterTest extends WebTestCase
{
/**
* @var BOMImporter
*/
protected $service;
protected function setUp(): void
{
parent::setUp();
//Get an service instance.
self::bootKernel();
$this->service = self::getContainer()->get(BOMImporter::class);
}
public function testImportFileIntoProject(): void
{
$inpute = $input = <<<CSV
"ID";"Bezeichner";"Footprint";"Stückzahl";"Bezeichnung";"Anbieter und Referenz";
1;"R19,R17";"R_0805_2012Metric_Pad1.20x1.40mm_HandSolder";2;"4.7k";Test;;
2;"D1";"D_DO-41_SOD81_P10.16mm_Horizontal";1;"1N5059";;;
3;"J3,J5";"JST_XH_B5B-XH-AM_1x05_P2.50mm_Vertical";2;"DISPLAY";;;
4;"C6";"CP_Radial_D6.3mm_P2.50mm";1;"47uF";;;
CSV;
$file = $this->createMock(File::class);
$file->method('getContent')->willReturn($input);
$project = new Project();
$this->assertCount(0, $project->getBOMEntries());
$bom_entries = $this->service->importFileIntoProject($file, $project, ['type' => 'kicad_pcbnew']);
$this->assertContainsOnlyInstancesOf(ProjectBOMEntry::class, $bom_entries);
$this->assertCount(4, $bom_entries);
//Check that the BOM entries are added to the project
$this->assertCount(4, $project->getBOMEntries());
}
public function testStringToBOMEntriesKiCADPCB(): void
{
//Test for german input
$input = <<<CSV
"ID";"Bezeichner";"Footprint";"Stückzahl";"Bezeichnung";"Anbieter und Referenz";
1;"R19,R17";"R_0805_2012Metric_Pad1.20x1.40mm_HandSolder";2;"4.7k";Test;;
2;"D1";"D_DO-41_SOD81_P10.16mm_Horizontal";1;"1N5059";;;
3;"J3,J5";"JST_XH_B5B-XH-AM_1x05_P2.50mm_Vertical";2;"DISPLAY";;;
4;"C6";"CP_Radial_D6.3mm_P2.50mm";1;"47uF";;;
CSV;
$bom = $this->service->stringToBOMEntries($input, ['type' => 'kicad_pcbnew']);
$this->assertContainsOnlyInstancesOf(ProjectBOMEntry::class, $bom);
$this->assertCount(4, $bom);
$this->assertEquals('R19,R17', $bom[0]->getMountnames());
$this->assertEquals(2.0, $bom[0]->getQuantity());
$this->assertSame('4.7k (R_0805_2012Metric_Pad1.20x1.40mm_HandSolder)', $bom[0]->getName());
$this->assertSame('Test', $bom[0]->getComment());
//Test for english input
$input = <<<CSV
"Id";"Designator";"Package";"Quantity";"Designation";"Supplier and ref";
1;"R19,R17";"R_0805_2012Metric_Pad1.20x1.40mm_HandSolder";2;"4.7k";Test;;
2;"D1";"D_DO-41_SOD81_P10.16mm_Horizontal";1;"1N5059";;;
3;"J3,J5";"JST_XH_B5B-XH-AM_1x05_P2.50mm_Vertical";2;"DISPLAY";;;
4;"C6";"CP_Radial_D6.3mm_P2.50mm";1;"47uF";;;
CSV;
$bom = $this->service->stringToBOMEntries($input, ['type' => 'kicad_pcbnew']);
$this->assertContainsOnlyInstancesOf(ProjectBOMEntry::class, $bom);
$this->assertCount(4, $bom);
$this->assertEquals('R19,R17', $bom[0]->getMountnames());
$this->assertEquals(2.0, $bom[0]->getQuantity());
$this->assertSame('4.7k (R_0805_2012Metric_Pad1.20x1.40mm_HandSolder)', $bom[0]->getName());
$this->assertSame('Test', $bom[0]->getComment());
}
public function testStringToBOMEntriesKiCADPCBError(): void
{
$input = <<<CSV
"ID";"Test";
1;"R19,R17";"R_0805_2012Metric_Pad1.20x1.40mm_HandSolder";2;"4.7k";Test;;
CSV;
$this->expectException(\UnexpectedValueException::class);
$this->service->stringToBOMEntries($input, ['type' => 'kicad_pcbnew']);
}
}

View file

@ -37,7 +37,7 @@ use Symfony\Component\Validator\ConstraintViolation;
class EntityImporterTest extends WebTestCase
{
/**
* @var AmountFormatter
* @var EntityImporter
*/
protected $service;

View file

@ -11133,5 +11133,59 @@ Element 3</target>
<target>If this option is selected, then all parts will be marked as "Needs review", no matter what was set in the data.</target>
</segment>
</unit>
<unit id="eVvGwjn" name="project.bom_import.flash.success">
<segment>
<source>project.bom_import.flash.success</source>
<target>Imported %count% BOM entries successfully.</target>
</segment>
</unit>
<unit id="5mJNteq" name="project.bom_import.type">
<segment>
<source>project.bom_import.type</source>
<target>Type</target>
</segment>
</unit>
<unit id="._nn_MY" name="project.bom_import.type.kicad_pcbnew">
<segment>
<source>project.bom_import.type.kicad_pcbnew</source>
<target>KiCAD Pcbnew BOM (CSV file)</target>
</segment>
</unit>
<unit id="iTVcfIk" name="project.bom_import.clear_existing_bom">
<segment>
<source>project.bom_import.clear_existing_bom</source>
<target>Clear existing BOM entries before importing</target>
</segment>
</unit>
<unit id="ijGhrbQ" name="project.bom_import.clear_existing_bom.help">
<segment>
<source>project.bom_import.clear_existing_bom.help</source>
<target>Selecting this option will remove all existing BOM entries in the project and overwrite them with the imported BOM file!</target>
</segment>
</unit>
<unit id="_FX5OYh" name="project.bom_import.flash.invalid_file">
<segment>
<source>project.bom_import.flash.invalid_file</source>
<target>File could not be imported. Please check that you have selected the right file type. Error message: %message%</target>
</segment>
</unit>
<unit id="1bpfKwW" name="project.bom_import.flash.invalid_entries">
<segment>
<source>project.bom_import.flash.invalid_entries</source>
<target>Validation error! Please check your data!</target>
</segment>
</unit>
<unit id="0oQjMQO" name="project.import_bom">
<segment>
<source>project.import_bom</source>
<target>Import BOM for project</target>
</segment>
</unit>
<unit id="sPZbhUL" name="project.edit.bom.import_bom">
<segment>
<source>project.edit.bom.import_bom</source>
<target>Import BOM</target>
</segment>
</unit>
</file>
</xliff>