Clean up the behavior tests (#5365)

* Speed up installs with pnp

* Move to a better relative path test

* Continue work on new test organization

* Move mjs test to new enhanced tests

* Move over last legacy test

* Update behavior e2e script

* Add first iteration of instructions to test readme

* Add some more bad instructions

* Split test command into multiple lines

* Use two workers (do not run in band)

* Bump install timeout

* No global pollution

* Fix PnP support via standard run mechanism

* Prevent circular serve command

* Silent yarn execs

* Fix common commands by using a shell

* Run with npx instead of Yarn

* Remove unused scripts

* Wait for localhost and fix script booting with PnP

* Don't pnp locally because links will cause module resolution to whine

* lint-staged is annoying

* Strip Yarn out of execa runs

* Set default license if none specified

* Don't impose worker limit

* Disable pnp for webpack messages

* Add missing dep...

fixtures/browser/graphql-with-mjs/__snapshots__/index.test.js.snap

-// Jest Snapshot v1, https://goo.gl/fbAQLP
-
-exports[`graphql with mjs entrypoint correctly bundles files in development 1`] = `"Pikachu"`;
-
-exports[`graphql with mjs entrypoint correctly bundles files in production 1`] = `"Pikachu"`;

fixtures/browser/graphql-with-mjs/index.test.js

-const {
-  bootstrap,
-  startDevelopmentServer,
-  startProductionServer,
-} = require('../../utils');
-const puppeteer = require('puppeteer');
-
-beforeEach(async () => {
-  await bootstrap({ directory: global.testDirectory, template: __dirname });
-  global.appDevPort = await startDevelopmentServer({
-    directory: global.testDirectory,
-  });
-  global.appProdPort = await startProductionServer({
-    directory: global.testDirectory,
-  });
-  // Wait for serve to boot up
-  await new Promise(resolve => setTimeout(resolve, 1000));
-});
-
-// https://github.com/facebook/create-react-app/issues/5234
-// https://github.com/facebook/create-react-app/pull/5258
-describe('graphql with mjs entrypoint', () => {
-  it('correctly bundles files in development', async () => {
-    const browser = await puppeteer.launch({ headless: true });
-    try {
-      const page = await browser.newPage();
-      await page.goto(`http://localhost:${global.appDevPort}/`);
-      await page.waitForSelector('.Pokemon-Name-Data');
-      const output = await page.evaluate(() => {
-        return Array.from(
-          document.getElementsByClassName('Pokemon-Name-Data')
-        ).pop().innerHTML;
-      });
-      expect(output).toMatchSnapshot();
-    } finally {
-      browser.close();
-    }
-  });
-
-  it('correctly bundles files in production', async () => {
-    const browser = await puppeteer.launch({ headless: true });
-    try {
-      const page = await browser.newPage();
-      await page.goto(`http://localhost:${global.appProdPort}/`);
-      await page.waitForSelector('.Pokemon-Name-Data');
-      const output = await page.evaluate(() => {
-        return Array.from(
-          document.getElementsByClassName('Pokemon-Name-Data')
-        ).pop().innerHTML;
-      });
-      expect(output).toMatchSnapshot();
-    } finally {
-      browser.close();
-    }
-  });
-});

fixtures/browser/jest.config.js

-module.exports = {
-  testEnvironment: 'node',
-  testMatch: ['**/*.test.js'],
-  testPathIgnorePatterns: ['/src/', 'node_modules'],
-  setupTestFrameworkScriptFile: './setupBrowserTests.js',
-  forceExit: true,
-};

fixtures/browser/setupBrowserTests.js

-const fs = require('fs-extra');
-const tempy = require('tempy');
-beforeEach(() => {
-  global.testDirectory = tempy.directory();
-  jest.setTimeout(1000 * 60 * 5);
-});
-afterEach(() => {
-  fs.removeSync(global.testDirectory);
-});

fixtures/output/jest.config.js

-module.exports = {
-  testEnvironment: 'node',
-  testMatch: ['**/*.test.js'],
-  setupTestFrameworkScriptFile: './setupOutputTests.js',
-};

fixtures/output/setupOutputTests.js

-beforeAll(() => {
-  jest.setTimeout(1000 * 60 * 5);
-});
-beforeEach(() => {
-  jest.setTimeout(1000 * 60 * 5);
-});

fixtures/output/webpack-message-formatting/index.test.js

-const {
-  bootstrap,
-  getOutputDevelopment,
-  getOutputProduction,
-} = require('../../utils');
-const fs = require('fs-extra');
-const path = require('path');
-const Semaphore = require('async-sema');
-const tempy = require('tempy');
-
-describe('webpack message formatting', () => {
-  const semaphore = new Semaphore(1, { capacity: Infinity });
-  let testDirectory;
-  beforeAll(async () => {
-    testDirectory = tempy.directory();
-    await bootstrap({ directory: testDirectory, template: __dirname });
-  });
-  beforeEach(async () => {
-    await semaphore.acquire();
-  });
-  afterEach(async () => {
-    fs.removeSync(path.join(testDirectory, 'src', 'App.js'));
-    semaphore.release();
-  });
-
-  it('formats babel syntax error', async () => {
-    fs.copySync(
-      path.join(__dirname, 'src', 'AppBabel.js'),
-      path.join(testDirectory, 'src', 'App.js')
-    );
-
-    const response = await getOutputProduction({ directory: testDirectory });
-    expect(response).toMatchSnapshot();
-  });
-
-  it('formats css syntax error', async () => {
-    fs.copySync(
-      path.join(__dirname, 'src', 'AppCss.js'),
-      path.join(testDirectory, 'src', 'App.js')
-    );
-
-    const response = await getOutputProduction({ directory: testDirectory });
-    expect(response).toMatchSnapshot();
-  });
-
-  it('formats unknown export', async () => {
-    fs.copySync(
-      path.join(__dirname, 'src', 'AppUnknownExport.js'),
-      path.join(testDirectory, 'src', 'App.js')
-    );
-
-    const response = await getOutputProduction({ directory: testDirectory });
-    expect(response).toMatchSnapshot();
-  });
-
-  it('formats aliased unknown export', async () => {
-    fs.copySync(
-      path.join(__dirname, 'src', 'AppAliasUnknownExport.js'),
-      path.join(testDirectory, 'src', 'App.js')
-    );
-
-    const response = await getOutputProduction({ directory: testDirectory });
-    expect(response).toMatchSnapshot();
-  });
-
-  it('formats no default export', async () => {
-    fs.copySync(
-      path.join(__dirname, 'src', 'AppNoDefault.js'),
-      path.join(testDirectory, 'src', 'App.js')
-    );
-
-    const response = await getOutputProduction({ directory: testDirectory });
-    expect(response).toMatchSnapshot();
-  });
-
-  it('formats missing package', async () => {
-    fs.copySync(
-      path.join(__dirname, 'src', 'AppMissingPackage.js'),
-      path.join(testDirectory, 'src', 'App.js')
-    );
-
-    const response = await getOutputProduction({ directory: testDirectory });
-    expect(response).toMatchSnapshot();
-  });
-
-  it('formats eslint warning', async () => {
-    fs.copySync(
-      path.join(__dirname, 'src', 'AppLintWarning.js'),
-      path.join(testDirectory, 'src', 'App.js')
-    );
-
-    const response = await getOutputProduction({ directory: testDirectory });
-    const sizeIndex = response.stdout.indexOf('File sizes after gzip');
-    if (sizeIndex !== -1) {
-      response.stdout = response.stdout.substring(0, sizeIndex);
-    }
-    expect(response).toMatchSnapshot();
-  });
-
-  it('formats eslint error', async () => {
-    fs.copySync(
-      path.join(__dirname, 'src', 'AppLintError.js'),
-      path.join(testDirectory, 'src', 'App.js')
-    );
-
-    const response = await getOutputProduction({ directory: testDirectory });
-    expect(response).toMatchSnapshot();
-  });
-
-  it('helps when users tries to use sass', async () => {
-    fs.copySync(
-      path.join(__dirname, 'src', 'AppSass.js'),
-      path.join(testDirectory, 'src', 'App.js')
-    );
-
-    const response = await getOutputProduction({ directory: testDirectory });
-    expect(response).toMatchSnapshot();
-  });
-
-  it('formats file not found error', async () => {
-    fs.copySync(
-      path.join(__dirname, 'src', 'AppUnknownFile.js'),
-      path.join(testDirectory, 'src', 'App.js')
-    );
-
-    const response = await getOutputProduction({ directory: testDirectory });
-    expect(response).toMatchSnapshot();
-  });
-
-  it('formats case sensitive path error', async () => {
-    fs.copySync(
-      path.join(__dirname, 'src', 'AppIncorrectCase.js'),
-      path.join(testDirectory, 'src', 'App.js')
-    );
-
-    const response = await getOutputDevelopment({ directory: testDirectory });
-    if (process.platform === 'darwin') {
-      expect(response.stderr).toMatch(
-        `Cannot find file: 'export5.js' does not match the corresponding name on disk: './src/Export5.js'.`
-      );
-    } else {
-      expect(response.stderr).not.toEqual(''); // TODO: figure out how we can test this on Linux/Windows
-      // I believe getting this working requires we tap into enhanced-resolve
-      // pipeline, which is debt we don't want to take on right now.
-    }
-  });
-
-  it('formats out of scope error', async () => {
-    fs.copySync(
-      path.join(__dirname, 'src', 'AppOutOfScopeImport.js'),
-      path.join(testDirectory, 'src', 'App.js')
-    );
-
-    const response = await getOutputProduction({ directory: testDirectory });
-    expect(response).toMatchSnapshot();
-  });
-});

fixtures/output/webpack-message-formatting/public/index.html

-<!DOCTYPE html>
-  <html lang="en">
-  <head>
-    <title>React App</title>
-  </head>
-  <body>
-    <div id="root"></div>
-  </body>
-</html>

fixtures/smoke/boostrap-sass/index.test.js

-const {
-  bootstrap,
-  isSuccessfulDevelopment,
-  isSuccessfulProduction,
-} = require('../../utils');
-beforeEach(async () => {
-  await bootstrap({ directory: global.testDirectory, template: __dirname });
-});
-
-describe('bootstrap sass', () => {
-  it('builds in development', async () => {
-    await isSuccessfulDevelopment({ directory: global.testDirectory });
-  });
-  it('builds in production', async () => {
-    await isSuccessfulProduction({ directory: global.testDirectory });
-  });
-});

fixtures/smoke/boostrap-sass/public/index.html

-<!DOCTYPE html>
-  <html lang="en">
-  <head>
-    <title>React App</title>
-  </head>
-  <body>
-    <div id="root"></div>
-  </body>
-</html>

fixtures/smoke/builds-with-multiple-runtimes/index.test.js

-const {
-  bootstrap,
-  isSuccessfulDevelopment,
-  isSuccessfulProduction,
-} = require('../../utils');
-beforeEach(async () => {
-  await bootstrap({ directory: global.testDirectory, template: __dirname });
-});
-
-describe('builds-with-multiple-runtimes', () => {
-  it('builds in development', async () => {
-    await isSuccessfulDevelopment({ directory: global.testDirectory });
-  });
-  it('builds in production', async () => {
-    await isSuccessfulProduction({ directory: global.testDirectory });
-  });
-});

fixtures/smoke/builds-with-multiple-runtimes/public/index.html

-<!DOCTYPE html>
-  <html lang="en">
-  <head>
-    <title>React App</title>
-  </head>
-  <body>
-    <div id="root"></div>
-  </body>
-</html>

fixtures/smoke/issue-5176-flow-class-properties/index.test.js

-const { bootstrap, isSuccessfulTest } = require('../../utils');
-beforeEach(async () => {
-  await bootstrap({ directory: global.testDirectory, template: __dirname });
-});
-
-describe('issue #5176 (flow class properties interaction)', () => {
-  it('passes tests', async () => {
-    await isSuccessfulTest({
-      directory: global.testDirectory,
-      jestEnvironment: 'node',
-    });
-  });
-});

fixtures/smoke/issue-5176-flow-class-properties/public/index.html

-<!DOCTYPE html>
-  <html lang="en">
-  <head>
-    <title>React App</title>
-  </head>
-  <body>
-    <div id="root"></div>
-  </body>
-</html>

fixtures/smoke/relative-paths/index.test.js

-const fs = require('fs-extra');
-const globby = require('globby');
-const path = require('path');
-const {
-  bootstrap,
-  isSuccessfulDevelopment,
-  isSuccessfulProduction,
-} = require('../../utils');
-beforeEach(async () => {
-  await bootstrap({ directory: global.testDirectory, template: __dirname });
-});
-
-describe('relative paths', () => {
-  // TODO: enable when development relative paths are supported
-  xit('builds in development', async () => {
-    await isSuccessfulDevelopment({ directory: global.testDirectory });
-  });
-  it('builds in production', async () => {
-    await isSuccessfulProduction({ directory: global.testDirectory });
-
-    const buildDir = path.join(global.testDirectory, 'build');
-    const cssFile = path.join(
-      buildDir,
-      globby.sync('**/*.css', { cwd: buildDir }).pop()
-    );
-    const svgFile = path.join(
-      buildDir,
-      globby.sync('**/*.svg', { cwd: buildDir }).pop()
-    );
-    const desiredPath = /url\((.+?)\)/
-      .exec(fs.readFileSync(cssFile, 'utf8'))
-      .pop();
-    expect(path.resolve(path.join(path.dirname(cssFile), desiredPath))).toBe(
-      path.resolve(svgFile)
-    );
-  });
-});

fixtures/smoke/relative-paths/public/index.html

-<!DOCTYPE html>
-  <html lang="en">
-  <head>
-    <title>React App</title>
-  </head>
-  <body>
-    <div id="root"></div>
-  </body>
-</html>

fixtures/smoke/setupSmokeTests.js

-const fs = require('fs-extra');
-const tempy = require('tempy');
-
-beforeEach(() => {
-  global.testDirectory = tempy.directory();
-  jest.setTimeout(1000 * 60 * 5);
-});
-afterEach(() => {
-  fs.removeSync(global.testDirectory);
-});

fixtures/utils.js

-const execa = require('execa');
-const fs = require('fs-extra');
-const getPort = require('get-port');
-const path = require('path');
-const os = require('os');
-const stripAnsi = require('strip-ansi');
-
-async function bootstrap({ directory, template }) {
-  const shouldInstallScripts = process.env.CI && process.env.CI !== 'false';
-  await Promise.all(
-    ['public/', 'src/', 'package.json'].map(async file =>
-      fs.copy(path.join(template, file), path.join(directory, file))
-    )
-  );
-  if (shouldInstallScripts) {
-    const packageJson = fs.readJsonSync(path.join(directory, 'package.json'));
-    packageJson.dependencies = Object.assign({}, packageJson.dependencies, {
-      'react-scripts': 'latest',
-    });
-    fs.writeJsonSync(path.join(directory, 'package.json'), packageJson);
-  }
-  await execa('yarnpkg', ['install', '--mutex', 'network'], { cwd: directory });
-  if (!shouldInstallScripts) {
-    fs.ensureSymlinkSync(
-      path.resolve(
-        path.join(
-          __dirname,
-          '..',
-          'packages',
-          'react-scripts',
-          'bin',
-          'react-scripts.js'
-        )
-      ),
-      path.join(directory, 'node_modules', '.bin', 'react-scripts')
-    );
-    await execa('yarnpkg', ['link', 'react-scripts'], { cwd: directory });
-  }
-}
-
-async function isSuccessfulDevelopment({ directory }) {
-  const { stdout, stderr } = await execa(
-    './node_modules/.bin/react-scripts',
-    ['start', '--smoke-test'],
-    {
-      cwd: directory,
-      env: { BROWSER: 'none', PORT: await getPort() },
-    }
-  );
-
-  if (!/Compiled successfully/.test(stdout)) {
-    throw new Error(`stdout: ${stdout}${os.EOL + os.EOL}stderr: ${stderr}`);
-  }
-}
-
-async function isSuccessfulProduction({ directory }) {
-  const { stdout, stderr } = await execa(
-    './node_modules/.bin/react-scripts',
-    ['build'],
-    {
-      cwd: directory,
-    }
-  );
-
-  if (!/Compiled successfully/.test(stdout)) {
-    throw new Error(`stdout: ${stdout}${os.EOL + os.EOL}stderr: ${stderr}`);
-  }
-}
-
-async function isSuccessfulTest({ directory, jestEnvironment = 'jsdom' }) {
-  await execa(
-    './node_modules/.bin/react-scripts',
-    ['test', '--env', jestEnvironment, '--ci'],
-    {
-      cwd: directory,
-      env: { CI: 'true' },
-    }
-  );
-}
-
-async function getOutputDevelopment({ directory, env = {} }) {
-  try {
-    const { stdout, stderr } = await execa(
-      './node_modules/.bin/react-scripts',
-      ['start', '--smoke-test'],
-      {
-        cwd: directory,
-        env: Object.assign(
-          {},
-          {
-            BROWSER: 'none',
-            PORT: await getPort(),
-            CI: 'false',
-            FORCE_COLOR: '0',
-          },
-          env
-        ),
-      }
-    );
-    return { stdout: stripAnsi(stdout), stderr: stripAnsi(stderr) };
-  } catch (err) {
-    return {
-      stdout: '',
-      stderr: stripAnsi(
-        err.message
-          .split(os.EOL)
-          .slice(2)
-          .join(os.EOL)
-      ),
-    };
-  }
-}
-
-async function getOutputProduction({ directory, env = {} }) {
-  try {
-    const { stdout, stderr } = await execa(
-      './node_modules/.bin/react-scripts',
-      ['build'],
-      {
-        cwd: directory,
-        env: Object.assign({}, { CI: 'false', FORCE_COLOR: '0' }, env),
-      }
-    );
-    return { stdout: stripAnsi(stdout), stderr: stripAnsi(stderr) };
-  } catch (err) {
-    return {
-      stdout: '',
-      stderr: stripAnsi(
-        err.message
-          .split(os.EOL)
-          .slice(2)
-          .join(os.EOL)
-      ),
-    };
-  }
-}
-
-async function startDevelopmentServer({ directory, env = {} }) {
-  const port = await getPort();
-  execa('./node_modules/.bin/react-scripts', ['start'], {
-    cwd: directory,
-    env: Object.assign(
-      {},
-      {
-        BROWSER: 'none',
-        PORT: port,
-        CI: 'false',
-        FORCE_COLOR: '0',
-      },
-      env
-    ),
-  });
-  return port;
-}
-
-async function startProductionServer({ directory, env = {} }) {
-  const port = await getPort();
-  await execa('./node_modules/.bin/react-scripts', ['build'], {
-    cwd: directory,
-    env: Object.assign({}, { CI: 'false' }, env),
-  });
-  execa('./node_modules/.bin/serve', ['-s', 'build', '-p', port], {
-    cwd: directory,
-  });
-  return port;
-}
-
-module.exports = {
-  bootstrap,
-  isSuccessfulDevelopment,
-  isSuccessfulProduction,
-  isSuccessfulTest,
-  getOutputDevelopment,
-  getOutputProduction,
-  startDevelopmentServer,
-  startProductionServer,
-};

package.json

@@ -19,7 +19,6 @@
     "compile:lockfile": "node tasks/compile-lockfile.js"
   },
   "devDependencies": {
-    "async-sema": "^2.1.3",
     "eslint": "5.6.0",
     "execa": "1.0.0",
     "fs-extra": "^7.0.0",
@@ -36,7 +35,8 @@
     "puppeteer": "^1.8.0",
     "strip-ansi": "^4.0.0",
     "svg-term-cli": "^2.1.1",
-    "tempy": "^0.2.1"
+    "tempy": "^0.2.1",
+    "wait-for-localhost": "2.0.1"
   },
   "husky": {
     "hooks": {

tasks/e2e-behavior.sh

@@ -95,14 +95,9 @@ git clean -df
 # Now that we have published them, run all tests as if they were released.
 # ******************************************************************************
 
-# Browser tests
-CI=true ./node_modules/.bin/jest --config fixtures/browser/jest.config.js
-
-# Smoke tests
-CI=true ./node_modules/.bin/jest --config fixtures/smoke/jest.config.js
-
-# Output tests
-CI=true ./node_modules/.bin/jest --config fixtures/output/jest.config.js
+# Run all tests
+cd test/
+CI=true ../node_modules/.bin/jest -w 2
 
 # Cleanup
 cleanup