Skip to content

Commit 127bda3

Browse files
IvanChepurnyiIvanChepurnyi
committed
feat: mysql2json import/export tool
new tool that allows quickly export and import data without modifying database structure feat(config): created JSONSchema for configuration files it allows to show nice errors messages for each incorrect value and defines defaults for configuration feat(config): added `includeTables` and `excludeTables` rules it is possible to define various table matching rules for the tool feat(config): added `concurrency` settings it is possible to control number of processes to use for parallel export/import feat(config): added `batchSize` setting it is possible to specify in configuration how many rows to import per batch feat(config): added `importMode` setting of `truncate` and `update` it is possible to specify if import replaces table completely or update current rows in it feat(command): added `mysql2jsonl export` command it exports data into JSONL file into desired directory with possibility to override concurrency feat(command): added `mysql2jsonl import` command it imports data form directory with JSONL files with possibility to override both concurrency and batch size feat(build): added Phar version to release pipeline it is possible to download now phar file with the tool, without installing it via composer
1 parent eb61723 commit 127bda3

File tree

87 files changed

+4727
-5
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

87 files changed

+4727
-5
lines changed

‎.github/workflows/php-package.yml

+52
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
name: PHP Package
2+
env:
3+
PHAR_TOOL_VERSION: 1.4.0
4+
PHAR_TOOL_REPOSITORY: clue/phar-composer
5+
on:
6+
push:
7+
pull_request:
8+
workflow_call:
9+
jobs:
10+
format-check:
11+
name: Check PSR12 Standarts
12+
runs-on: ubuntu-24.04
13+
steps:
14+
- uses: actions/checkout@v4
15+
- name: Setup PHP
16+
uses: shivammathur/setup-php@v2
17+
with:
18+
php-version: 8.2
19+
tools: composer:v2
20+
- run: composer install
21+
shell: bash
22+
- run: composer format:check
23+
shell: bash
24+
tests:
25+
name: Run Tests
26+
runs-on: ubuntu-24.04
27+
strategy:
28+
matrix:
29+
php-version:
30+
- 8.2
31+
- 8.3
32+
- 8.4
33+
steps:
34+
- uses: actions/checkout@v4
35+
- name: Setup PHP
36+
uses: shivammathur/setup-php@v2
37+
with:
38+
php-version: ${{ matrix.php-version }}
39+
tools: composer:v2
40+
coverage: xdebug3
41+
- run: composer install
42+
shell: bash
43+
- run: composer test
44+
shell: bash
45+
- name: Download build package
46+
run: gh release download v${{ env.PHAR_TOOL_VERSION }} -R=${{ env.PHAR_TOOL_REPOSITORY }}
47+
env:
48+
GH_TOKEN: ${{ github.token }}
49+
- name: Build package
50+
run: |
51+
chmod +x ./phar-composer-${{ env.PHAR_TOOL_VERSION }}.phar
52+
./phar-composer-${{ env.PHAR_TOOL_VERSION }}.phar build ./ mysql2json

‎.github/workflows/release-please.yml

+53
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
on:
2+
push:
3+
branches:
4+
- main
5+
env:
6+
PHAR_TOOL_VERSION: 1.4.0
7+
PHAR_TOOL_REPOSITORY: clue/phar-composer
8+
9+
permissions:
10+
contents: write
11+
pull-requests: write
12+
13+
name: Create Release
14+
15+
jobs:
16+
verify-release:
17+
uses: ./.github/workflows/php-package.yml
18+
release-please:
19+
needs: verify-release
20+
runs-on: ubuntu-24.04
21+
outputs:
22+
releases_created: ${{ steps.release.outputs.release_created }}
23+
tag: ${{ steps.release.outputs.tag_name }}
24+
steps:
25+
- uses: googleapis/release-please-action@v4
26+
id: release
27+
with:
28+
release-type: php
29+
upload_phar:
30+
needs: release-please
31+
runs-on: ubuntu-24.04
32+
if: ${{ needs.release-please.outputs.releases_created == true }}
33+
steps:
34+
- uses: actions/checkout@v4
35+
with:
36+
ref: ${{ needs.release-please.outputs.tag }}
37+
- name: Setup PHP
38+
uses: shivammathur/setup-php@v2
39+
with:
40+
php-version: 8.2
41+
tools: composer:v2
42+
- name: Download build package
43+
run: gh release download v${{ env.PHAR_TOOL_VERSION }} -R=${{ env.PHAR_TOOL_REPOSITORY }}
44+
- name: Build package
45+
run: |
46+
chmod +x ./phar-composer-${{ env.PHAR_TOOL_VERSION }}.phar
47+
./phar-composer-${{ env.PHAR_TOOL_VERSION }}.phar build ./ mysql2json
48+
env:
49+
GH_TOKEN: ${{ github.token }}
50+
- name: Upload package
51+
run: gh release upload ${{ needs.release-please.outputs.tag }} mysql2json
52+
env:
53+
GH_TOKEN: ${{ github.token }}

‎.gitignore

+3-5
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
1-
composer.phar
21
/vendor/
3-
4-
# Commit your application's lock file https://getcomposer.org/doc/01-basic-usage.md#commit-your-composer-lock-file-to-version-control
5-
# You may choose to ignore a library lock file http://getcomposer.org/doc/02-libraries.md#lock-file
6-
# composer.lock
2+
/composer.lock
3+
/.idea
4+
/.phpunit.cache

‎.release-please-manifest.json

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
".": "1.0.0"
3+
}

‎bin/mysql2jsonl

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#!/usr/bin/env php
2+
<?php
3+
4+
use EcomDev\MySQL2JSONL\Command\ExportCommand;
5+
use EcomDev\MySQL2JSONL\Command\ImportCommand;
6+
use Symfony\Component\Console\Application;
7+
8+
require_once __DIR__ . '/../vendor/autoload.php';
9+
10+
$application = new Application(
11+
basename(__FILE__),
12+
'1.0.0'
13+
);
14+
15+
$application->addCommands([
16+
new ExportCommand(),
17+
new ImportCommand(),
18+
]);
19+
20+
$application->run();

‎composer.json

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
{
2+
"name": "ecomdev/mysql-to-jsonl",
3+
"description": "Export/Import JSONL files as data for MySQL tables",
4+
"type": "library",
5+
"require": {
6+
"php": "~8.2",
7+
"amphp/amp": "~3.0",
8+
"amphp/parallel": "~v2.3.1",
9+
"revolt/event-loop": "~1.0",
10+
"symfony/console": "~7.2",
11+
"justinrainbow/json-schema": "^6.0",
12+
"ext-pdo": "*"
13+
},
14+
"require-dev": {
15+
"squizlabs/php_codesniffer": "^3.0",
16+
"phpunit/phpunit": "^11.5",
17+
"brianium/paratest": "^7.7",
18+
"ecomdev/testcontainers-magento-data":"~1.2"
19+
},
20+
"license": [
21+
"MIT"
22+
],
23+
"keywords": [
24+
"mysql", "jsonl", "backup"
25+
],
26+
"autoload": {
27+
"psr-4": {
28+
"EcomDev\\MySQL2JSONL\\": "src/"
29+
}
30+
},
31+
"autoload-dev": {
32+
"files": [
33+
"tests/fixtures.php"
34+
],
35+
"psr-4": {
36+
"EcomDev\\MySQL2JSONL\\": "tests/"
37+
}
38+
},
39+
"scripts": {
40+
"test": "XDEBUG_MODE=coverage paratest --coverage-text",
41+
"test:single-threaded": "XDEBUG_MODE=coverage phpunit --coverage-text",
42+
"format:check": "phpcs",
43+
"format:write": "phpcbf"
44+
},
45+
"bin": [
46+
"bin/mysql2jsonl"
47+
],
48+
"$schema": "https://getcomposer.org/schema.json"
49+
}

‎phpcs.xml

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
<?xml version="1.0"?>
2+
<ruleset name="EcomDev">
3+
<description>The coding standard for EcomDev packages.</description>
4+
5+
<file>src</file>
6+
<file>tests</file>
7+
<exclude-pattern>tests/fixtures.php</exclude-pattern>
8+
9+
<arg value="np"/>
10+
<arg name="colors"/>
11+
12+
<!-- Include some additional sniffs from the Generic standard -->
13+
<rule ref="Generic.Commenting.DocComment"/>
14+
15+
<!-- Use Unix newlines -->
16+
<rule ref="Generic.Files.LineEndings">
17+
<properties>
18+
<property name="eolChar" value="\n"/>
19+
</properties>
20+
</rule>
21+
22+
<rule ref="PSR2"/>
23+
<rule ref="PSR12"/>
24+
</ruleset>

‎phpunit.xml

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<phpunit xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="https://schema.phpunit.de/11.5/phpunit.xsd" bootstrap="vendor/autoload.php" executionOrder="depends,defects"
3+
beStrictAboutOutputDuringTests="true"
4+
colors="true"
5+
cacheDirectory=".phpunit.cache"
6+
displayDetailsOnTestsThatTriggerWarnings="true"
7+
displayDetailsOnTestsThatTriggerNotices="true"
8+
displayDetailsOnTestsThatTriggerDeprecations="true"
9+
>
10+
<testsuites>
11+
<testsuite name="default">
12+
<directory>tests</directory>
13+
</testsuite>
14+
</testsuites>
15+
<source>
16+
<include>
17+
<directory>src</directory>
18+
</include>
19+
</source>
20+
<coverage ignoreDeprecatedCodeUnits="true">
21+
</coverage>
22+
</phpunit>

‎release-please-config.json

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{
2+
"packages": {
3+
".": {
4+
"changelog-path": "CHANGELOG.md",
5+
"release-type": "php",
6+
"bump-minor-pre-major": true,
7+
"bump-patch-for-minor-pre-major": false,
8+
"draft": false,
9+
"include-v-in-tag": false,
10+
"prerelease": false
11+
}
12+
},
13+
"$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json"
14+
}

0 commit comments

Comments
 (0)