test: migrate module_api to system tests (#20265)

Co-authored-by: Ryan Manuel <[email protected]>

circle.yml

@@ -1705,39 +1705,6 @@ jobs:
             yarn cypress:run --project /tmp/cypress-test-tiny --record
       - store-npm-logs
 
-  test-binary-and-npm-against-other-projects:
-    <<: *defaults
-    parameters:
-      <<: *defaultsParameters
-      resource_class:
-        type: string
-        default: medium
-    resource_class: << parameters.resource_class >>
-    steps:
-      # needs uploaded NPM and test binary
-      - restore_cached_workspace
-      - run: ls -la
-      # make sure JSON files with uploaded urls are present
-      - run: ls -la binary-url.json npm-package-url.json
-      - run: cat binary-url.json
-      - run: cat npm-package-url.json
-      - run:
-          # install NPM from unique urls
-          name: Install Cypress Binary in Dummy Package
-          command: |
-            node scripts/test-unique-npm-and-binary.js \
-              --npm npm-package-url.json \
-              --binary binary-url.json \
-              --cwd /tmp/testing
-      - run:
-          name: Running other test projects with new NPM package and binary
-          command: |
-            node scripts/test-other-projects.js \
-              --npm npm-package-url.json \
-              --binary binary-url.json \
-              --provider circle
-      - store-npm-logs
-
   test-npm-module-and-verify-binary:
     <<: *defaults
     steps:
@@ -2278,11 +2245,6 @@ linux-workflow: &linux-workflow
     - test-binary-against-kitchensink:
         requires:
           - create-build-artifacts
-    - test-binary-and-npm-against-other-projects:
-        context: test-runner:trigger-test-jobs
-        <<: *mainBuildFilters
-        requires:
-          - create-build-artifacts
     - test-npm-module-and-verify-binary:
         <<: *mainBuildFilters
         requires:
@@ -2373,13 +2335,6 @@ mac-workflow: &mac-workflow
         requires:
           - darwin-build
 
-    - test-binary-and-npm-against-other-projects:
-        context: test-runner:trigger-test-jobs
-        name: darwin-test-binary-and-npm-against-other-projects
-        executor: mac
-        requires:
-          - darwin-create-build-artifacts
-
 windows-workflow: &windows-workflow
   jobs:
     - node_modules_install:
@@ -2419,14 +2374,6 @@ windows-workflow: &windows-workflow
         requires:
           - windows-build
 
-    - test-binary-and-npm-against-other-projects:
-        context: test-runner:trigger-test-jobs
-        name: windows-test-binary-and-npm-against-other-projects
-        executor: windows
-        resource_class: windows.medium
-        requires:
-          - windows-create-build-artifacts
-
 workflows:
   linux:
     <<: *linux-workflow

cli/package.json

@@ -9,8 +9,6 @@
     "build": "node ./scripts/build.js",
     "dtslint": "dtslint types",
     "postinstall": "node ./scripts/post-install.js",
-    "prerelease": "yarn build",
-    "release": "cd build && releaser --no-node --no-changelog",
     "size": "t=\"cypress-v0.0.0.tgz\"; yarn pack --filename \"${t}\"; wc -c \"${t}\"; tar tvf \"${t}\"; rm \"${t}\";",
     "test": "yarn test-unit",
     "test-debug": "node --inspect-brk $(yarn bin mocha)",

guides/release-process.md

@@ -111,14 +111,7 @@ In the following instructions, "X.Y.Z" is used to denote the [next version of Cy
         - Go into a project, run a quick test, make sure things look right
     - Install the new version into an established project and run the tests there
         - [cypress-realworld-app](https://github.com/cypress-io/cypress-realworld-app) uses yarn and represents a typical consumer implementation.
-    - Optionally, do more thorough tests:
-        - Trigger test projects from the command line (if you have the appropriate permissions)
-
-        ```shell
-        node scripts/test-other-projects.js --npm [email protected] --binary X.Y.Z
-        ```
-
-        - Test the new version of Cypress against the Cypress dashboard repo.
+    - Optionally, do more thorough tests, for example test the new version of Cypress against the Cypress dashboard repo.
 
 7. Confirm that every issue labeled [stage: pending release](https://github.com/cypress-io/cypress/issues?q=label%3A%22stage%3A+pending+release%22+is%3Aclosed) has a ZenHub release set. **Tip:** there is a command in [`release-automations`](https://github.com/cypress-io/release-automations)'s `issues-in-release` tool to list and check such issues. Without a ZenHub release issues will not be included in the right changelog.
 

guides/testing-other-projects.md

@@ -4,10 +4,10 @@ In `develop`, `master`, and any other branch configured in [`circle.yml`](../cir
 
 Two main strategies are used to spawn these test projects:
 
-1. Local CI
+1. `test-binary-against-repo` jobs
 2. Remote CI
 
-## Local CI
+## `test-binary-against-repo` jobs
 
 A number of CI jobs in `circle.yml` clone test projects and run tests as part of `cypress-io/cypress`'s CI pipeline.
 
@@ -17,30 +17,6 @@ Similarly to "Remote CI" test projects, Local CI test projects will attempt to c
 
 One advantage to local CI is that it does not require creating commits to another repo.
 
-## Remote CI
+## `binary-system-tests`
 
-After the production binary and NPM package are build and uploaded in CI, [`/scripts/test-other-projects.js`](../scripts/test-other-projects.js) is run as part of the `test-other-projects` `circle.yml` step.
-
-This script creates commits inside of several test projects (hence "Remote CI") in order to trigger a realistic, continous-integration test of Cypress.
-
-For a list of the projects, see the definition of `_PROVIDERS` in [`/scripts/binary/bump.js`](../scripts/binary/bump.js).
-
-For each project and operating system combo in `_PROVIDERS`, the script:
-
-1. Creates a commit to the test project's GitHub repo using the API. [An example of such a commit.](https://github.com/cypress-io/cypress-test-tiny/commit/5b39f3f43f6b7598f0d57cffcba71a7048d1d809)
-    * Note the commit is specifically for `linux`, and only the `linux-tests` job runs to completion.
-    * If a branch exists that is named after the [next version](./next-version.md) (`X.Y.Z`), the commit will be made to that branch.
-        * This is useful to test a release's breaking changes or new features against an example project without having to have the project's main branch in a broken state.
-    * Otherwise, the default branch is used for the commit.
-2. Creates a status check in this GitHub repo (`cypress-io/cypress`) and marks it `pending`.
-3. Waits for the test project's CI workflow to finish running.
-    * Each test project is configured to use [`@cypress/commit-message-install`](https://github.com/cypress-io/commit-message-install) to configure the exact test required via the information in the commit message.
-    * Each test project is configured to update the `pending` CI job in `cypress-io/cypress` to a `success` when the CI workflow successfully finishes.
-
-These tests add coverage to the Cypress code base by:
-
-* Providing a super-close-to-real-world usage of Cypress (i.e. installing fresh from an NPM package and running in a bare repo using the repo's CI setup)
-* Testing in a variety of environments
-  * Different Node.js versions
-  * Different operating systems
-  * A multitude of CI providers
+System tests in `/system-tests/test-binary` are run against the built Cypress App in CI. For more details, see the [README](../system-tests/README.md).

package.json

@@ -14,7 +14,6 @@
     "binary-zip": "node ./scripts/binary.js zip",
     "build": "lerna run build --stream --ignore create-cypress-tests && lerna run build --stream --scope create-cypress-tests",
     "build-prod": "lerna run build-prod --stream --ignore create-cypress-tests && lerna run build-prod --stream --scope create-cypress-tests",
-    "bump": "node ./scripts/binary.js bump",
     "check-node-version": "node scripts/check-node-version.js",
     "check-terminal": "node scripts/check-terminal.js",
     "clean": "lerna run clean --parallel --no-bail || echo 'ok, errors while cleaning'",
@@ -73,7 +72,6 @@
     "nvm": "0.0.4"
   },
   "devDependencies": {
-    "@cypress/bumpercar": "2.0.12",
     "@cypress/commit-message-install": "3.1.3",
     "@cypress/env-or-json-file": "2.0.0",
     "@cypress/github-commit-status-check": "1.5.0",
@@ -157,14 +155,12 @@
     "lint-staged": "11.1.2",
     "listr2": "3.8.3",
     "lodash": "^4.17.21",
-    "make-empty-github-commit": "cypress-io/make-empty-github-commit#4a592aedb776ba2f4cc88979055315a53eec42ee",
     "minimist": "1.2.5",
     "mocha": "3.5.3",
     "mocha-banner": "1.1.2",
     "mocha-junit-reporter": "2.0.0",
     "mocha-multi-reporters": "1.1.7",
     "mock-fs": "5.1.1",
-    "parse-github-repo-url": "1.4.1",
     "patch-package": "6.4.7",
     "plist": "3.0.4",
     "pluralize": "8.0.0",