BAP-10564: Provide ability to automatically split functional tests in…

…to equal test suites

1) functional-x-of-y test suites were removed

Those suites caused a lot of issues when we miss some tests in suites (often issue for extensions), and they become broken afterward. For now, please use --testsuite=functional and use folder argument or filter tests using built-in PHPUnit functionality in case you need to run some specific tests, see examples below:

```
bin/phpunit --testsuite=functional
bin/phpunit --testsuite=functional vendor/oro/platform/src/Oro/Bundle/TestFrameworkBundle/Tests/Functional
bin/phpunit --testsuite=functional --filter="TestFrameworkBundle\\\\Tests\\\\Functional"
```

The bundle-based approach used on Travis and Jenkins. We collect a list of existing bundles and run them in parallel threads one by one.

2) GNU Parallel applied for Travis instead of custom shell scripts.

GNU Parallel job report printed in `if [ -z "$TRAVIS_SKIP" ]; then ./.travis/$TESTSUITE.sh after_script; fi` section.

Few tips how to use it for development (run commands from application folder). The 2017 year edition required, check it using `parallel --version` and do not forget to record environment using `parallel --record-env` command

# dependencies
COMPOSER=dev.json composer install -o --profile

PHPCS:
find -L vendor/oro -type f -name "*.php" | parallel --no-notice --gnu -k --lb --env _ --xargs --joblog app/logs/parallel.log php bin/phpcs {} -p --encoding=utf-8 --extensions=php --standard=vendor/oro/platform/build/phpcs.xml;

PHPMD for OroPlatform and OroCRM:
find -L vendor/oro -type f -name "*.php" | parallel --no-notice --gnu -k --lb --env _ --xargs --joblog app/logs/parallel.log 'files="{}"; php bin/phpmd ${files// /,} text vendor/oro/platform/build/phpmd.xml --suffixes php'

PHPMD for OroCommerce:
find -L vendor/oro -type f -name "*.php" | grep -i commerce | parallel --no-notice --gnu -k --lb --env _ --xargs --joblog app/logs/parallel.log 'files="{}"; php bin/phpmd ${files// /,} text vendor/oro/commerce/build_config/phpmd.xml --suffixes php'

# dependencies
npm install --prefix=vendor/oro/platform/build/

JSCS:
find -L vendor/oro -type f -name "*.js" | grep -iv node_modules | parallel --no-notice --gnu -k --lb --env _ --xargs --joblog app/logs/parallel.log 'vendor/oro/platform/build/node_modules/.bin/jscs {} --config=vendor/oro/platform/build/.jscsrc'

JSHINT:
find -L vendor/oro -type f -name "*.js" | grep -iv node_modules | parallel --no-notice --gnu -k --lb --env _ --xargs --joblog app/logs/parallel.log 'vendor/oro/platform/build/node_modules/.bin/jshint {} --config=vendor/oro/platform/build/.jshintrc --exclude-path=vendor/oro/platform/build/.jshintignore'

3) \Oro\Bundle\TestFrameworkBundle\Tests\Functional\InstallerTest added to check that installers are up to date with migrations and no uncovered migration left. Executed for installation only (jobs with upgrades skipped) and excluded from test suites by default. To run it locally use next command

bin/phpunit --testsuite functional --group=install

4) bootstrap.php files for PHPUnit were removed. Symfony PHPUnit Bridge handles bootstrapping now.

5) phpunit.xml.dist were actualized. Templates for those files created in package/platform/build folder

6) phpcpd was upgraded to dev-master branch, locked using hash until it's next release, allows us to use regexp in excludes, like

bin/phpcpd --min-lines 25 --verbose --progress --regexps-exclude=Migrations/Schema/,Entity/ vendor/oro/commerce

7) Segfaults. Switching to per bundle approach reduces segfault chances. If you still have segfaults on your build just add GDB=true variable to failed job and uncomment gdb package, you will get segfault report and will be able to detect wrong code.

8) New dependency, phploc/phploc under BSD 3-clause license, used to generate coverage report, will be applied in a few weeks to our CI flow

.gitignore

@@ -8,3 +8,4 @@ composer.lock
 phpunit.xml
 *~
 node_modules
+/components

.travis.yml

@@ -14,20 +14,12 @@ php:
 cache:
   directories:
     - $HOME/.composer
-    - $TRAVIS_BUILD_DIR/vendor
 
 before_script:
   - phpenv config-rm xdebug.ini
-  - phpenv config-add travis.php.ini
   - composer self-update
-  - composer config minimum-stability dev
-  - composer config prefer-stable true
-  - composer config repositories.oro composer https://packagist.orocrm.com
-  - composer require --no-update --dev "phpunit/phpunit:5.7.*"
-  - composer require --no-update --dev "squizlabs/php_codesniffer:2.8.*"
-  - composer require --no-update --dev "phpmd/phpmd:2.6.*"
   - composer global require fxp/composer-asset-plugin
-  - travis_wait composer update --prefer-dist --optimize-autoloader --no-interaction  --no-suggest --prefer-stable
+  - travis_wait composer install --prefer-dist --optimize-autoloader --no-interaction  --no-suggest
   - set +e; DIFF=$(git diff --name-only --diff-filter=ACMR $TRAVIS_COMMIT_RANGE | grep -e ".*\.php$"); set -e;
 
 script:

UPGRADE-2.3.md

@@ -22,3 +22,7 @@ SearchBundle
 ------------
 - Class `Oro\Bundle\SearchBundle\EventListener\ReindexDemoDataListener` was replaced with `Oro\Bundle\SearchBundle\EventListener\ReindexDemoDataFixturesListener`
 - Service `oro_search.event_listener.reindex_demo_data` was replaced with `oro_search.migration.demo_data_fixtures_listener.reindex`
+
+TestFrameworkBundle
+-------------------
+- Class `TestListener` namespace added, use `Oro\Bundle\TestFrameworkBundle\Test\TestListener` instead

build/.jshintignore

@@ -1,3 +1,4 @@
+**/.bin/**
 ../../**/node_modules/**
 ../../**/public/lib/**
 ../../**/public/*/vendors/**

build/phpunit_app.xml.dist

@@ -0,0 +1,137 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!-- https://phpunit.de/manual/current/en/appendixes.configuration.html -->
+<phpunit
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:noNamespaceSchemaLocation="http://schema.phpunit.de/5.7/phpunit.xsd"
+        backupGlobals="false"
+        backupStaticAttributes="false"
+        bootstrap="vendor/autoload.php"
+        cacheTokens="true"
+        stopOnError="false"
+        stopOnFailure="false"
+        stopOnWarning="false"
+        stopOnIncomplete="false"
+        stopOnRisky="false"
+        stopOnSkipped="false"
+        failOnRisky="false"
+        failOnWarning="true"
+        beStrictAboutChangesToGlobalState="false"
+        beStrictAboutOutputDuringTests="false"
+        beStrictAboutResourceUsageDuringSmallTests="false"
+        beStrictAboutTestsThatDoNotTestAnything="false"
+        beStrictAboutTodoAnnotatedTests="false"
+        beStrictAboutCoversAnnotation="false"
+        checkForUnintentionallyCoveredCode="false"
+        enforceTimeLimit="false"
+        verbose="false">
+
+    <groups>
+        <exclude>
+            <group>dist</group>
+            <group>install</group>
+        </exclude>
+    </groups>
+    <testsuites>
+        <testsuite name="unit">
+            <directory>vendor/oro/*/Tests/Unit</directory>
+            <directory>vendor/oro/*/*/Tests/Unit</directory>
+            <directory>vendor/oro/*/*/*/Tests/Unit</directory>
+            <directory>vendor/oro/*/*/*/*/Tests/Unit</directory>
+            <directory>vendor/oro/*/*/*/*/*/Tests/Unit</directory>
+        </testsuite>
+        <testsuite name="functional">
+            <directory>vendor/oro/*/Tests/Functional</directory>
+            <directory>vendor/oro/*/*/Tests/Functional</directory>
+            <directory>vendor/oro/*/*/*/Tests/Functional</directory>
+            <directory>vendor/oro/*/*/*/*/Tests/Functional</directory>
+            <directory>vendor/oro/*/*/*/*/*/Tests/Functional</directory>
+        </testsuite>
+        <testsuite name="selenium">
+            <directory>vendor/oro/*/Tests/Selenium</directory>
+            <directory>vendor/oro/*/*/Tests/Selenium</directory>
+            <directory>vendor/oro/*/*/*/Tests/Selenium</directory>
+            <directory>vendor/oro/*/*/*/*/Tests/Selenium</directory>
+            <directory>vendor/oro/*/*/*/*/*/Tests/Selenium</directory>
+        </testsuite>
+    </testsuites>
+    <php>
+        <env name="SYMFONY_ENV" value="test"/>
+        <env name="SYMFONY_DEBUG" value="0"/>
+        <env name="SYMFONY_DEPRECATIONS_HELPER" value="weak" />
+        <ini name="error_reporting" value="-1"/>
+        <ini name="memory_limit" value="-1"/>
+        <server name="KERNEL_DIR" value="app/"/>
+        <const name="PHPUNIT_LOAD_LIMIT" value="1.2"/>
+        <const name="PHPUNIT_PAGE_LIMIT" value="0.5"/>
+        <const name="PHPUNIT_TESTSUITE_BROWSER_PATH_WINNT" value="C:\Dev\phantomjs-1.9.0-windows\phantomjs.exe"/>
+        <const name="PHPUNIT_TESTSUITE_BROWSER_PATH_LINUX" value="/usr/bin/phantomjs"/>
+        <!--<const name="PHPUNIT_TESTSUITE_BROWSER_PATH_LINUX" value="/usr/bin/google-chrome"/>-->
+        <!--<const name="PHPUNIT_TESTSUITE_BROWSER_PATH_LINUX" value="/usr/bin/firefox"/>-->
+        <const name="PHPUNIT_TESTSUITE" value="true"/>
+        <const name="PHPUNIT_TESTSUITE_EXTENSION_SELENIUM_HOST" value="127.0.0.1"/>
+        <const name="PHPUNIT_TESTSUITE_EXTENSION_SELENIUM_PORT" value="4444"/>
+        <const name="PHPUNIT_TESTSUITE_EXTENSION_SELENIUM2_BROWSER" value="phantomjs"/>
+        <!--<const name="PHPUNIT_TESTSUITE_EXTENSION_SELENIUM2_BROWSER" value="chrome"/>-->
+        <!--<const name="PHPUNIT_TESTSUITE_EXTENSION_SELENIUM2_BROWSER" value="firefox"/>-->
+        <const name="PHPUNIT_TESTSUITE_EXTENSION_SELENIUM_TESTS_URL" value="http://localhost.com"/>
+        <const name="PHPUNIT_TESTSUITE_EXTENSION_SELENIUM_TESTS_URL_COVERAGE"
+               value="http://localhost.com/bundles/orotestframework/scripts/phpunit_coverage.php"/>
+        <const name="MAX_EXECUTION_TIME" value="240000"/>
+        <const name="TIME_OUT" value="5000"/>
+        <const name="viewportWIDTH" value="1900"/>
+        <const name="viewportHEIGHT" value="1080"/>
+        <const name="PHPUNIT_TESTSUITE_EXTENSION_SELENIUM_PATH_LOGS" value="app/log"/>
+        <const name="PHPUNIT_TESTSUITE_EXTENSION_SELENIUM_LOGIN" value="admin"/>
+        <const name="PHPUNIT_TESTSUITE_EXTENSION_SELENIUM_PASS" value="admin"/>
+    </php>
+    <listeners>
+        <listener class="JohnKary\PHPUnit\Listener\SpeedTrapListener">
+            <arguments>
+                <array>
+                    <element key="slowThreshold">
+                        <integer>1000</integer>
+                    </element>
+                </array>
+            </arguments>
+        </listener>
+        <listener class="Symfony\Bridge\PhpUnit\SymfonyTestsListener"/>
+        <listener class="MyBuilder\PhpunitAccelerator\TestListener"/>
+    </listeners>
+    <filter>
+        <whitelist>
+            <directory>vendor/oro</directory>
+            <exclude>
+                <directory>vendor/oro/*/Command</directory>
+                <directory>vendor/oro/*/*/Command</directory>
+                <directory>vendor/oro/*/*/*/Command</directory>
+                <directory>vendor/oro/*/*/*/*/Command</directory>
+                <directory>vendor/oro/*/*/*/*/*/Command</directory>
+
+                <directory>vendor/oro/*/Controller</directory>
+                <directory>vendor/oro/*/*/Controller</directory>
+                <directory>vendor/oro/*/*/*/Controller</directory>
+                <directory>vendor/oro/*/*/*/*/Controller</directory>
+                <directory>vendor/oro/*/*/*/*/*/Controller</directory>
+
+                <directory>vendor/oro/*/Entity/Repository</directory>
+                <directory>vendor/oro/*/*/Entity/Repository</directory>
+                <directory>vendor/oro/*/*/*/Entity/Repository</directory>
+                <directory>vendor/oro/*/*/*/*/Entity/Repository</directory>
+                <directory>vendor/oro/*/*/*/*/*/Entity/Repository</directory>
+
+                <directory>vendor/oro/*/Migrations</directory>
+                <directory>vendor/oro/*/*/Migrations</directory>
+                <directory>vendor/oro/*/*/*/Migrations</directory>
+                <directory>vendor/oro/*/*/*/*/Migrations</directory>
+                <directory>vendor/oro/*/*/*/*/*/Migrations</directory>
+
+                <directory>vendor/oro/*/Tests</directory>
+                <directory>vendor/oro/*/*/Tests</directory>
+                <directory>vendor/oro/*/*/*/Tests</directory>
+                <directory>vendor/oro/*/*/*/*/Tests</directory>
+                <directory>vendor/oro/*/*/*/*/*/Tests</directory>
+            </exclude>
+        </whitelist>
+    </filter>
+</phpunit>

build/phpunit_package.xml.dist

@@ -0,0 +1,63 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!-- https://phpunit.de/manual/current/en/appendixes.configuration.html -->
+<phpunit
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:noNamespaceSchemaLocation="http://schema.phpunit.de/5.7/phpunit.xsd"
+        backupGlobals="false"
+        backupStaticAttributes="false"
+        bootstrap="vendor/autoload.php"
+        cacheTokens="true"
+        stopOnError="false"
+        stopOnFailure="false"
+        stopOnWarning="false"
+        stopOnIncomplete="false"
+        stopOnRisky="false"
+        stopOnSkipped="false"
+        failOnRisky="false"
+        failOnWarning="true"
+        beStrictAboutChangesToGlobalState="false"
+        beStrictAboutOutputDuringTests="false"
+        beStrictAboutResourceUsageDuringSmallTests="false"
+        beStrictAboutTestsThatDoNotTestAnything="false"
+        beStrictAboutTodoAnnotatedTests="false"
+        beStrictAboutCoversAnnotation="false"
+        checkForUnintentionallyCoveredCode="false"
+        enforceTimeLimit="false"
+        verbose="false">
+
+    <testsuites>
+        <testsuite name="unit">
+            <directory>Tests/Unit</directory>
+            <directory>src/*/Tests/Unit</directory>
+            <directory>src/*/*/Tests/Unit</directory>
+            <directory>src/*/*/*/Tests/Unit</directory>
+            <directory>src/*/*/*/*/Tests/Unit</directory>
+            <directory>src/*/*/*/*/*/Tests/Unit</directory>
+            <directory>src/*/*/*/*/*/*/Tests/Unit</directory>
+            <directory>src/*/*/*/*/*/*/*/Tests/Unit</directory>
+        </testsuite>
+    </testsuites>
+
+    <php>
+        <env name="SYMFONY_ENV" value="test"/>
+        <env name="SYMFONY_DEBUG" value="0"/>
+        <env name="SYMFONY_ENV" value="test"/>
+        <env name="SYMFONY_DEPRECATIONS_HELPER" value="weak" />
+        <ini name="error_reporting" value="-1"/>
+        <ini name="memory_limit" value="-1"/>
+    </php>
+    <listeners>
+        <listener class="JohnKary\PHPUnit\Listener\SpeedTrapListener">
+            <arguments>
+                <array>
+                    <element key="slowThreshold">
+                        <integer>1000</integer>
+                    </element>
+                </array>
+            </arguments>
+        </listener>
+        <listener class="Symfony\Bridge\PhpUnit\SymfonyTestsListener"/>
+        <listener class="MyBuilder\PhpunitAccelerator\TestListener"/>
+    </listeners>
+</phpunit>

composer.json

@@ -126,9 +126,18 @@
         "mybuilder/phpunit-accelerator": "1.2.*",
         "squizlabs/php_codesniffer": "2.8.*",
         "phpmd/phpmd": "2.6.*",
-        "sebastian/phpcpd": "2.0.*",
+        "sebastian/phpcpd": "dev-master#cff7f36c2ae89d59987de25d14fd41a72dd4a703 as 3.1.0",
+        "phpunit/phpcov": "3.1.*",
         "symfony/phpunit-bridge": "3.2.*"
     },
+    "repositories": {
+        "oro": {
+            "type": "composer",
+            "url": "https://packagist.orocrm.com"
+        }
+    },
+    "minimum-stability": "dev",
+    "prefer-stable": true,
     "extra": {
         "branch-alias": {
             "dev-master": "2.x-dev"

phpunit.xml.dist

@@ -1,39 +1,63 @@
 <?xml version="1.0" encoding="UTF-8"?>
 
-<!-- http://www.phpunit.de/manual/current/en/appendixes.configuration.html -->
+<!-- https://phpunit.de/manual/current/en/appendixes.configuration.html -->
 <phpunit
-        backupGlobals               = "false"
-        backupStaticAttributes      = "false"
-        colors                      = "false"
-        convertErrorsToExceptions   = "true"
-        convertNoticesToExceptions  = "true"
-        convertWarningsToExceptions = "true"
-        processIsolation            = "false"
-        stopOnFailure               = "false"
-        syntaxCheck                 = "false"
-        bootstrap                   = "tests/bootstrap.php"
->
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:noNamespaceSchemaLocation="http://schema.phpunit.de/5.7/phpunit.xsd"
+        backupGlobals="false"
+        backupStaticAttributes="false"
+        bootstrap="vendor/autoload.php"
+        cacheTokens="true"
+        stopOnError="false"
+        stopOnFailure="false"
+        stopOnWarning="false"
+        stopOnIncomplete="false"
+        stopOnRisky="false"
+        stopOnSkipped="false"
+        failOnRisky="false"
+        failOnWarning="true"
+        beStrictAboutChangesToGlobalState="false"
+        beStrictAboutOutputDuringTests="false"
+        beStrictAboutResourceUsageDuringSmallTests="false"
+        beStrictAboutTestsThatDoNotTestAnything="false"
+        beStrictAboutTodoAnnotatedTests="false"
+        beStrictAboutCoversAnnotation="false"
+        checkForUnintentionallyCoveredCode="false"
+        enforceTimeLimit="false"
+        verbose="false">
+
     <testsuites>
-        <testsuite name="Project Unit Tests">
-            <directory suffix="Test.php">src/Oro/Bundle/*Bundle/Tests/Unit</directory>
+        <testsuite name="unit">
+            <directory>Tests/Unit</directory>
+            <directory>src/*/Tests/Unit</directory>
+            <directory>src/*/*/Tests/Unit</directory>
+            <directory>src/*/*/*/Tests/Unit</directory>
+            <directory>src/*/*/*/*/Tests/Unit</directory>
+            <directory>src/*/*/*/*/*/Tests/Unit</directory>
+            <directory>src/*/*/*/*/*/*/Tests/Unit</directory>
+            <directory>src/*/*/*/*/*/*/*/Tests/Unit</directory>
         </testsuite>
     </testsuites>
 
-    <!--
     <php>
-        <server name="KERNEL_DIR" value="/path/to/your/app/" />
+        <env name="SYMFONY_ENV" value="test"/>
+        <env name="SYMFONY_DEBUG" value="0"/>
+        <env name="SYMFONY_ENV" value="test"/>
+        <env name="SYMFONY_DEPRECATIONS_HELPER" value="weak" />
+        <ini name="error_reporting" value="-1"/>
+        <ini name="memory_limit" value="-1"/>
     </php>
-    -->
-    <filter>
-        <whitelist>
-            <directory>src</directory>
-            <exclude>
-                <directory>vendor</directory>
-                <directory>src/Oro/Bundle/*Bundle/DataFixtures</directory>
-                <directory>src/Oro/Bundle/*Bundle/Resources</directory>
-                <directory>src/Oro/Bundle/*Bundle/Tests</directory>
-                <directory>vendor</directory>
-            </exclude>
-        </whitelist>
-    </filter>
+    <listeners>
+        <listener class="JohnKary\PHPUnit\Listener\SpeedTrapListener">
+            <arguments>
+                <array>
+                    <element key="slowThreshold">
+                        <integer>1000</integer>
+                    </element>
+                </array>
+            </arguments>
+        </listener>
+        <listener class="Symfony\Bridge\PhpUnit\SymfonyTestsListener"/>
+        <listener class="MyBuilder\PhpunitAccelerator\TestListener"/>
+    </listeners>
 </phpunit>