switch to just triggering specific integration test jobs

.github/workflows/codeql.yml

@@ -1,6 +1,6 @@
 name: "CodeQL action"
 
-on: [push, pull_request]
+on: [pull_request]
 
 jobs:
   build:

.github/workflows/integration-testing.yml

@@ -3,45 +3,490 @@ name: "Integration Testing"
 on: [push, pull_request]
 
 jobs:
-  multi-language-repo_test-autodetect-languages:
+  # multi-language-repo_test-autodetect-languages:
+  #   runs-on: ubuntu-latest
+
+  #   steps:
+  #   - uses: actions/checkout@v2
+  #   - name: Move codeql-action
+  #     shell: bash
+  #     run: |
+  #       mkdir ../action
+  #       mv * .github ../action/
+  #       mv ../action/tests/multi-language-repo/{*,.github} .
+  #   - uses: ./../action/init
+  #   - name: Build code
+  #     shell: bash
+  #     run: ./build.sh
+  #   - uses: ./../action/analyze
+  #     env:
+  #       TEST_MODE: true
+  #   - run: |
+  #       cd "$RUNNER_TEMP/codeql_databases"
+  #       # List all directories as there will be precisely one directory per database
+  #       # but there may be other files in this directory such as query suites.
+  #       if [ "$(ls -d */ | wc -l)" != 6 ] || \
+  #          [[ ! -d cpp ]] || \
+  #          [[ ! -d csharp ]] || \
+  #          [[ ! -d go ]] || \
+  #          [[ ! -d java ]] || \
+  #          [[ ! -d javascript ]] || \
+  #          [[ ! -d python ]]; then
+  #         echo "Did not find expected number of databases. Database dir contains: $(ls)"
+  #         exit 1
+  #       fi
+
+  # multi-language-repo_test-custom-queries-and-remote-config:
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       os: [ubuntu-latest, windows-latest, macos-latest]
+  #   runs-on: ${{ matrix.os }}
+
+  #   steps:
+  #   - uses: actions/checkout@v2
+  #   - name: Move codeql-action
+  #     shell: bash
+  #     run: |
+  #       mkdir ../action
+  #       mv * .github ../action/
+  #       mv ../action/tests/multi-language-repo/{*,.github} .
+  #   - uses: ./../action/init
+  #     with:
+  #       languages: cpp,csharp,java,javascript,python
+  #       config-file: github/codeql-action/tests/multi-language-repo/.github/codeql/custom-queries.yml@${{ github.sha }}
+  #   - name: Build code
+  #     shell: bash
+  #     run: ./build.sh
+  #   - uses: ./../action/analyze
+  #     env:
+  #       TEST_MODE: true
+
+  # # Currently is not possible to analyze Go in conjunction with other languages in macos
+  # multi-language-repo_test-go-custom-queries:
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       os: [ubuntu-latest, windows-latest, macos-latest]
+  #   runs-on: ${{ matrix.os }}
+
+  #   steps:
+  #   - uses: actions/setup-go@v2
+  #     if: ${{ matrix.os ==  'macos-latest' }}
+  #     with:
+  #       go-version: '^1.13.1'
+  #   - uses: actions/checkout@v2
+  #   - name: Move codeql-action
+  #     shell: bash
+  #     run: |
+  #       mkdir ../action
+  #       mv * .github ../action/
+  #       mv ../action/tests/multi-language-repo/{*,.github} .
+  #   - uses: ./../action/init
+  #     with:
+  #       languages: go
+  #       config-file: ./.github/codeql/custom-queries.yml
+  #   - name: Build code
+  #     shell: bash
+  #     run: ./build.sh
+  #   - uses: ./../action/analyze
+  #     env:
+  #       TEST_MODE: true
+
+  # multi-language-repo_rubocop:
+  #   runs-on: ubuntu-latest
+
+  #   steps:
+  #   - uses: actions/checkout@v2
+  #   - name: Move codeql-action
+  #     shell: bash
+  #     run: |
+  #       mkdir ../action
+  #       mv * .github ../action/
+  #       mv ../action/tests/multi-language-repo/{*,.github} .
+  #   - name: Set up Ruby
+  #     uses: ruby/setup-ruby@v1
+  #     with:
+  #       ruby-version: 2.6
+  #   - name: Install Code Scanning integration
+  #     run: bundle add code-scanning-rubocop --version 0.3.0 --skip-install
+  #   - name: Install dependencies
+  #     run: bundle install
+  #   - name: Rubocop run
+  #     run: |
+  #       bash -c "
+  #         bundle exec rubocop --require code_scanning --format CodeScanning::SarifFormatter -o rubocop.sarif
+  #         [[ $? -ne 2 ]]
+  #       "
+  #   - uses: ./../action/upload-sarif
+  #     with:
+  #       sarif_file: rubocop.sarif
+  #     env:
+  #       TEST_MODE: true
+
+  # test-proxy:
+  #   runs-on: ubuntu-latest
+  #   container:
+  #     image: ubuntu:18.04
+  #     options: --dns 127.0.0.1
+  #   services:
+  #     squid-proxy:
+  #       image: datadog/squid:latest
+  #       ports:
+  #         - 3128:3128
+  #   env:
+  #     https_proxy: http://squid-proxy:3128
+  #   steps:
+  #   - uses: actions/checkout@v2
+  #   - name: Move codeql-action
+  #     shell: bash
+  #     run: |
+  #       mkdir ../action
+  #       mv * .github ../action/
+  #       mv ../action/tests/multi-language-repo/{*,.github} .
+  #   - uses: ./../action/init
+  #     with:
+  #       languages: javascript
+  #   - uses: ./../action/analyze
+  #     env:
+  #       TEST_MODE: true
+  
+  # runner-analyze-javascript-ubuntu:
+  #   runs-on: ubuntu-latest
+
+  #   steps:
+  #   - uses: actions/checkout@v2
+
+  #   - name: Build runner
+  #     run: |
+  #       cd runner
+  #       npm install
+  #       npm run build-runner
+
+  #   - name: Run init
+  #     run: |
+  #       # Pass --config-file here, but not for other jobs in this workflow.
+  #       # This means we're testing the config file parsing in the runner
+  #       # but not slowing down all jobs unnecessarily as it doesn't add much
+  #       # testing the parsing on different operating systems and languages.
+  #       runner/dist/codeql-runner-linux init --repository $GITHUB_REPOSITORY --languages javascript --config-file ./.github/codeql/codeql-config.yml --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
+
+  #   - name: Run analyze
+  #     run: |
+  #       runner/dist/codeql-runner-linux analyze --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
+  #     env:
+  #       TEST_MODE: true
+  
+  # runner-analyze-javascript-windows:
+  #   runs-on: windows-latest
+
+  #   steps:
+  #   - uses: actions/checkout@v2
+
+  #   - name: Build runner
+  #     run: |
+  #       cd runner
+  #       npm install
+  #       npm run build-runner
+
+  #   - name: Run init
+  #     run: |
+  #       runner/dist/codeql-runner-win.exe init --repository $Env:GITHUB_REPOSITORY --languages javascript --github-url $Env:GITHUB_SERVER_URL --github-auth ${{ github.token }}
+
+  #   - name: Run analyze
+  #     run: |
+  #       runner/dist/codeql-runner-win.exe analyze --repository $Env:GITHUB_REPOSITORY --commit $Env:GITHUB_SHA --ref $Env:GITHUB_REF --github-url $Env:GITHUB_SERVER_URL --github-auth ${{ github.token }}
+  #     env:
+  #       TEST_MODE: true
+  
+  # runner-analyze-javascript-macos:
+  #   runs-on: macos-latest
+
+  #   steps:
+  #   - uses: actions/checkout@v2
+
+  #   - name: Build runner
+  #     run: |
+  #       cd runner
+  #       npm install
+  #       npm run build-runner
+
+  #   - name: Run init
+  #     run: |
+  #       runner/dist/codeql-runner-macos init --repository $GITHUB_REPOSITORY --languages javascript --config-file ./.github/codeql/codeql-config.yml --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
+
+  #   - name: Run analyze
+  #     run: |
+  #       runner/dist/codeql-runner-macos analyze --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
+  #     env:
+  #       TEST_MODE: true
+  
+  # runner-analyze-csharp-ubuntu:
+  #   runs-on: ubuntu-latest
+
+  #   steps:
+  #   - uses: actions/checkout@v2
+
+  #   - name: Move codeql-action
+  #     shell: bash
+  #     run: |
+  #       mkdir ../action
+  #       mv * .github ../action/
+  #       mv ../action/tests/multi-language-repo/{*,.github} .
+
+  #   - name: Build runner
+  #     run: |
+  #       cd ../action/runner
+  #       npm install
+  #       npm run build-runner
+
+  #   - name: Run init
+  #     run: |
+  #       ../action/runner/dist/codeql-runner-linux init --repository $GITHUB_REPOSITORY --languages csharp --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
+
+  #   - name: Build code
+  #     run: |
+  #       . ./codeql-runner/codeql-env.sh
+  #       dotnet build
+
+  #   - name: Run analyze
+  #     run: |
+  #       ../action/runner/dist/codeql-runner-linux analyze --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
+  #     env:
+  #       TEST_MODE: true
+  
+  # runner-analyze-csharp-windows:
+  #   runs-on: windows-latest
+
+  #   steps:
+  #   - uses: actions/checkout@v2
+
+  #   - name: Move codeql-action
+  #     shell: bash
+  #     run: |
+  #       mkdir ../action
+  #       mv * .github ../action/
+  #       mv ../action/tests/multi-language-repo/{*,.github} .
+
+  #   - name: Build runner
+  #     run: |
+  #       cd ../action/runner
+  #       npm install
+  #       npm run build-runner
+
+  #   - name: Run init
+  #     run: |
+  #       ../action/runner/dist/codeql-runner-win.exe init --repository $Env:GITHUB_REPOSITORY --languages csharp --github-url $Env:GITHUB_SERVER_URL --github-auth ${{ github.token }}
+
+  #   - name: Build code
+  #     shell: powershell
+  #     run: |
+  #       cat ./codeql-runner/codeql-env.sh | Invoke-Expression
+  #       dotnet build
+
+  #   - name: Run analyze
+  #     run: |
+  #       ../action/runner/dist/codeql-runner-win.exe analyze --repository $Env:GITHUB_REPOSITORY --commit $Env:GITHUB_SHA --ref $Env:GITHUB_REF --github-url $Env:GITHUB_SERVER_URL --github-auth ${{ github.token }}
+  #     env:
+  #       TEST_MODE: true
+
+  # runner-analyze-csharp-macos:
+  #   runs-on: macos-latest
+
+  #   steps:
+  #   - uses: actions/checkout@v2
+
+  #   - name: Move codeql-action
+  #     shell: bash
+  #     run: |
+  #       mkdir ../action
+  #       mv * .github ../action/
+  #       mv ../action/tests/multi-language-repo/{*,.github} .
+
+  #   - name: Build runner
+  #     run: |
+  #       cd ../action/runner
+  #       npm install
+  #       npm run build-runner
+
+  #   - name: Run init
+  #     run: |
+  #       ../action/runner/dist/codeql-runner-macos init --repository $GITHUB_REPOSITORY --languages csharp --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
+
+  #   - name: Build code
+  #     shell: bash
+  #     run: |
+  #       . ./codeql-runner/codeql-env.sh
+  #       dotnet build
+
+  #   - name: Run analyze
+  #     run: |
+  #       ../action/runner/dist/codeql-runner-macos analyze --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
+  #     env:
+  #       TEST_MODE: true
+
+  
+  # runner-analyze-csharp-autobuild-ubuntu:
+  #   runs-on: ubuntu-latest
+
+  #   steps:
+  #   - uses: actions/checkout@v2
+
+  #   - name: Move codeql-action
+  #     shell: bash
+  #     run: |
+  #       mkdir ../action
+  #       mv * .github ../action/
+  #       mv ../action/tests/multi-language-repo/{*,.github} .
+
+  #   - name: Build runner
+  #     run: |
+  #       cd ../action/runner
+  #       npm install
+  #       npm run build-runner
+
+  #   - name: Run init
+  #     run: |
+  #       ../action/runner/dist/codeql-runner-linux init --repository $GITHUB_REPOSITORY --languages csharp --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
+
+  #   - name: Build code
+  #     run: |
+  #       ../action/runner/dist/codeql-runner-linux autobuild
+
+  #   - name: Run analyze
+  #     run: |
+  #       ../action/runner/dist/codeql-runner-linux analyze --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
+  #     env:
+  #       TEST_MODE: true
+  
+  # runner-analyze-csharp-autobuild-windows:
+  #   runs-on: windows-latest
+
+  #   steps:
+  #   - uses: actions/checkout@v2
+
+  #   - name: Move codeql-action
+  #     shell: bash
+  #     run: |
+  #       mkdir ../action
+  #       mv * .github ../action/
+  #       mv ../action/tests/multi-language-repo/{*,.github} .
+
+  #   - name: Build runner
+  #     run: |
+  #       cd ../action/runner
+  #       npm install
+  #       npm run build-runner
+
+  #   - name: Run init
+  #     run: |
+  #       ../action/runner/dist/codeql-runner-win.exe init --repository $Env:GITHUB_REPOSITORY --languages csharp --github-url $Env:GITHUB_SERVER_URL --github-auth ${{ github.token }}
+
+  #   - name: Build code
+  #     shell: powershell
+  #     run: |
+  #       ../action/runner/dist/codeql-runner-win.exe autobuild
+
+  #   - name: Run analyze
+  #     run: |
+  #       ../action/runner/dist/codeql-runner-win.exe analyze --repository $Env:GITHUB_REPOSITORY --commit $Env:GITHUB_SHA --ref $Env:GITHUB_REF --github-url $Env:GITHUB_SERVER_URL --github-auth ${{ github.token }}
+  #     env:
+  #       TEST_MODE: true
+
+  # runner-analyze-csharp-autobuild-macos:
+  #   runs-on: macos-latest
+
+  #   steps:
+  #   - uses: actions/checkout@v2
+
+  #   - name: Move codeql-action
+  #     shell: bash
+  #     run: |
+  #       mkdir ../action
+  #       mv * .github ../action/
+  #       mv ../action/tests/multi-language-repo/{*,.github} .
+
+  #   - name: Build runner
+  #     run: |
+  #       cd ../action/runner
+  #       npm install
+  #       npm run build-runner
+
+  #   - name: Run init
+  #     run: |
+  #       ../action/runner/dist/codeql-runner-macos init --repository $GITHUB_REPOSITORY --languages csharp --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
+
+  #   - name: Build code
+  #     shell: bash
+  #     run: |
+  #       ../action/runner/dist/codeql-runner-macos autobuild
+
+  #   - name: Run analyze
+  #     run: |
+  #       ../action/runner/dist/codeql-runner-macos analyze --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
+  #     env:
+  #       TEST_MODE: true
+
+  # runner-upload-sarif:
+  #   runs-on: ubuntu-latest
+
+  #   steps:
+  #   - uses: actions/checkout@v2
+
+  #   - name: Build runner
+  #     run: |
+  #       cd runner
+  #       npm install
+  #       npm run build-runner
+
+  #   - name: Upload with runner
+  #     run: |
+  #       # Deliberately don't use TEST_MODE here. This is specifically testing
+  #       # the compatibility with the API.
+  #       runner/dist/codeql-runner-linux upload --sarif-file src/testdata/empty-sarif.sarif --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
+
+# Test the behaviour when asked to scan a compiled language but no code is compiled
+  java:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
-    - name: Move codeql-action
-      shell: bash
-      run: |
-        mkdir ../action
-        mv * .github ../action/
-        mv ../action/tests/multi-language-repo/{*,.github} .
-    - uses: ./../action/init
-    - name: Build code
-      shell: bash
-      run: ./build.sh
-    - uses: ./../action/analyze
-      env:
-        TEST_MODE: true
-    - run: |
-        cd "$RUNNER_TEMP/codeql_databases"
-        # List all directories as there will be precisely one directory per database
-        # but there may be other files in this directory such as query suites.
-        if [ "$(ls -d */ | wc -l)" != 6 ] || \
-           [[ ! -d cpp ]] || \
-           [[ ! -d csharp ]] || \
-           [[ ! -d go ]] || \
-           [[ ! -d java ]] || \
-           [[ ! -d javascript ]] || \
-           [[ ! -d python ]]; then
-          echo "Did not find expected number of databases. Database dir contains: $(ls)"
-          exit 1
-        fi
+      - uses: actions/checkout@v2
+      - name: Move codeql-action
+        shell: bash
+        run: |
+          mkdir ../action
+          shopt -s dotglob
+          mv * ../action/
+          # Do not copy anything back, so there's nothing left to scan
+      - uses: ./../action/init
+        with:
+          languages: java
+      - uses: ./../action/analyze
+        env:
+          TEST_MODE: true
+  
+  # Test the behaviour when asked to scan a non-compiled language but there is no code to scan
+  javascript:
+    runs-on: ubuntu-latest
 
-  multi-language-repo_test-custom-queries-and-remote-config:
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
-    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Move codeql-action
+        shell: bash
+        run: |
+          mkdir ../action
+          shopt -s dotglob
+          mv * ../action/
+          # Do not copy anything back, so there's nothing left to scan
+      - uses: ./../action/init
+        with:
+          languages: javascript
+      - uses: ./../action/analyze
+        env:
+          TEST_MODE: true
+
+  matcher-catches-out-of-memory-error:
+    runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v2
@@ -59,388 +504,7 @@ jobs:
       shell: bash
       run: ./build.sh
     - uses: ./../action/analyze
-      env:
-        TEST_MODE: true
-
-  # Currently is not possible to analyze Go in conjunction with other languages in macos
-  multi-language-repo_test-go-custom-queries:
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
-    runs-on: ${{ matrix.os }}
-
-    steps:
-    - uses: actions/setup-go@v2
-      if: ${{ matrix.os ==  'macos-latest' }}
       with:
-        go-version: '^1.13.1'
-    - uses: actions/checkout@v2
-    - name: Move codeql-action
-      shell: bash
-      run: |
-        mkdir ../action
-        mv * .github ../action/
-        mv ../action/tests/multi-language-repo/{*,.github} .
-    - uses: ./../action/init
-      with:
-        languages: go
-        config-file: ./.github/codeql/custom-queries.yml
-    - name: Build code
-      shell: bash
-      run: ./build.sh
-    - uses: ./../action/analyze
+        ram: 10
       env:
         TEST_MODE: true
-
-  multi-language-repo_rubocop:
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v2
-    - name: Move codeql-action
-      shell: bash
-      run: |
-        mkdir ../action
-        mv * .github ../action/
-        mv ../action/tests/multi-language-repo/{*,.github} .
-    - name: Set up Ruby
-      uses: ruby/setup-ruby@v1
-      with:
-        ruby-version: 2.6
-    - name: Install Code Scanning integration
-      run: bundle add code-scanning-rubocop --version 0.3.0 --skip-install
-    - name: Install dependencies
-      run: bundle install
-    - name: Rubocop run
-      run: |
-        bash -c "
-          bundle exec rubocop --require code_scanning --format CodeScanning::SarifFormatter -o rubocop.sarif
-          [[ $? -ne 2 ]]
-        "
-    - uses: ./../action/upload-sarif
-      with:
-        sarif_file: rubocop.sarif
-      env:
-        TEST_MODE: true
-
-  test-proxy:
-    runs-on: ubuntu-latest
-    container:
-      image: ubuntu:18.04
-      options: --dns 127.0.0.1
-    services:
-      squid-proxy:
-        image: datadog/squid:latest
-        ports:
-          - 3128:3128
-    env:
-      https_proxy: http://squid-proxy:3128
-    steps:
-    - uses: actions/checkout@v2
-    - name: Move codeql-action
-      shell: bash
-      run: |
-        mkdir ../action
-        mv * .github ../action/
-        mv ../action/tests/multi-language-repo/{*,.github} .
-    - uses: ./../action/init
-      with:
-        languages: javascript
-    - uses: ./../action/analyze
-      env:
-        TEST_MODE: true
-  
-  runner-analyze-javascript-ubuntu:
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Build runner
-      run: |
-        cd runner
-        npm install
-        npm run build-runner
-
-    - name: Run init
-      run: |
-        # Pass --config-file here, but not for other jobs in this workflow.
-        # This means we're testing the config file parsing in the runner
-        # but not slowing down all jobs unnecessarily as it doesn't add much
-        # testing the parsing on different operating systems and languages.
-        runner/dist/codeql-runner-linux init --repository $GITHUB_REPOSITORY --languages javascript --config-file ./.github/codeql/codeql-config.yml --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
-
-    - name: Run analyze
-      run: |
-        runner/dist/codeql-runner-linux analyze --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
-      env:
-        TEST_MODE: true
-  
-  runner-analyze-javascript-windows:
-    runs-on: windows-latest
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Build runner
-      run: |
-        cd runner
-        npm install
-        npm run build-runner
-
-    - name: Run init
-      run: |
-        runner/dist/codeql-runner-win.exe init --repository $Env:GITHUB_REPOSITORY --languages javascript --github-url $Env:GITHUB_SERVER_URL --github-auth ${{ github.token }}
-
-    - name: Run analyze
-      run: |
-        runner/dist/codeql-runner-win.exe analyze --repository $Env:GITHUB_REPOSITORY --commit $Env:GITHUB_SHA --ref $Env:GITHUB_REF --github-url $Env:GITHUB_SERVER_URL --github-auth ${{ github.token }}
-      env:
-        TEST_MODE: true
-  
-  runner-analyze-javascript-macos:
-    runs-on: macos-latest
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Build runner
-      run: |
-        cd runner
-        npm install
-        npm run build-runner
-
-    - name: Run init
-      run: |
-        runner/dist/codeql-runner-macos init --repository $GITHUB_REPOSITORY --languages javascript --config-file ./.github/codeql/codeql-config.yml --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
-
-    - name: Run analyze
-      run: |
-        runner/dist/codeql-runner-macos analyze --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
-      env:
-        TEST_MODE: true
-  
-  runner-analyze-csharp-ubuntu:
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Move codeql-action
-      shell: bash
-      run: |
-        mkdir ../action
-        mv * .github ../action/
-        mv ../action/tests/multi-language-repo/{*,.github} .
-
-    - name: Build runner
-      run: |
-        cd ../action/runner
-        npm install
-        npm run build-runner
-
-    - name: Run init
-      run: |
-        ../action/runner/dist/codeql-runner-linux init --repository $GITHUB_REPOSITORY --languages csharp --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
-
-    - name: Build code
-      run: |
-        . ./codeql-runner/codeql-env.sh
-        dotnet build
-
-    - name: Run analyze
-      run: |
-        ../action/runner/dist/codeql-runner-linux analyze --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
-      env:
-        TEST_MODE: true
-  
-  runner-analyze-csharp-windows:
-    runs-on: windows-latest
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Move codeql-action
-      shell: bash
-      run: |
-        mkdir ../action
-        mv * .github ../action/
-        mv ../action/tests/multi-language-repo/{*,.github} .
-
-    - name: Build runner
-      run: |
-        cd ../action/runner
-        npm install
-        npm run build-runner
-
-    - name: Run init
-      run: |
-        ../action/runner/dist/codeql-runner-win.exe init --repository $Env:GITHUB_REPOSITORY --languages csharp --github-url $Env:GITHUB_SERVER_URL --github-auth ${{ github.token }}
-
-    - name: Build code
-      shell: powershell
-      run: |
-        cat ./codeql-runner/codeql-env.sh | Invoke-Expression
-        dotnet build
-
-    - name: Run analyze
-      run: |
-        ../action/runner/dist/codeql-runner-win.exe analyze --repository $Env:GITHUB_REPOSITORY --commit $Env:GITHUB_SHA --ref $Env:GITHUB_REF --github-url $Env:GITHUB_SERVER_URL --github-auth ${{ github.token }}
-      env:
-        TEST_MODE: true
-
-  runner-analyze-csharp-macos:
-    runs-on: macos-latest
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Move codeql-action
-      shell: bash
-      run: |
-        mkdir ../action
-        mv * .github ../action/
-        mv ../action/tests/multi-language-repo/{*,.github} .
-
-    - name: Build runner
-      run: |
-        cd ../action/runner
-        npm install
-        npm run build-runner
-
-    - name: Run init
-      run: |
-        ../action/runner/dist/codeql-runner-macos init --repository $GITHUB_REPOSITORY --languages csharp --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
-
-    - name: Build code
-      shell: bash
-      run: |
-        . ./codeql-runner/codeql-env.sh
-        dotnet build
-
-    - name: Run analyze
-      run: |
-        ../action/runner/dist/codeql-runner-macos analyze --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
-      env:
-        TEST_MODE: true
-
-  
-  runner-analyze-csharp-autobuild-ubuntu:
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Move codeql-action
-      shell: bash
-      run: |
-        mkdir ../action
-        mv * .github ../action/
-        mv ../action/tests/multi-language-repo/{*,.github} .
-
-    - name: Build runner
-      run: |
-        cd ../action/runner
-        npm install
-        npm run build-runner
-
-    - name: Run init
-      run: |
-        ../action/runner/dist/codeql-runner-linux init --repository $GITHUB_REPOSITORY --languages csharp --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
-
-    - name: Build code
-      run: |
-        ../action/runner/dist/codeql-runner-linux autobuild
-
-    - name: Run analyze
-      run: |
-        ../action/runner/dist/codeql-runner-linux analyze --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
-      env:
-        TEST_MODE: true
-  
-  runner-analyze-csharp-autobuild-windows:
-    runs-on: windows-latest
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Move codeql-action
-      shell: bash
-      run: |
-        mkdir ../action
-        mv * .github ../action/
-        mv ../action/tests/multi-language-repo/{*,.github} .
-
-    - name: Build runner
-      run: |
-        cd ../action/runner
-        npm install
-        npm run build-runner
-
-    - name: Run init
-      run: |
-        ../action/runner/dist/codeql-runner-win.exe init --repository $Env:GITHUB_REPOSITORY --languages csharp --github-url $Env:GITHUB_SERVER_URL --github-auth ${{ github.token }}
-
-    - name: Build code
-      shell: powershell
-      run: |
-        ../action/runner/dist/codeql-runner-win.exe autobuild
-
-    - name: Run analyze
-      run: |
-        ../action/runner/dist/codeql-runner-win.exe analyze --repository $Env:GITHUB_REPOSITORY --commit $Env:GITHUB_SHA --ref $Env:GITHUB_REF --github-url $Env:GITHUB_SERVER_URL --github-auth ${{ github.token }}
-      env:
-        TEST_MODE: true
-
-  runner-analyze-csharp-autobuild-macos:
-    runs-on: macos-latest
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Move codeql-action
-      shell: bash
-      run: |
-        mkdir ../action
-        mv * .github ../action/
-        mv ../action/tests/multi-language-repo/{*,.github} .
-
-    - name: Build runner
-      run: |
-        cd ../action/runner
-        npm install
-        npm run build-runner
-
-    - name: Run init
-      run: |
-        ../action/runner/dist/codeql-runner-macos init --repository $GITHUB_REPOSITORY --languages csharp --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
-
-    - name: Build code
-      shell: bash
-      run: |
-        ../action/runner/dist/codeql-runner-macos autobuild
-
-    - name: Run analyze
-      run: |
-        ../action/runner/dist/codeql-runner-macos analyze --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
-      env:
-        TEST_MODE: true
-
-  runner-upload-sarif:
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Build runner
-      run: |
-        cd runner
-        npm install
-        npm run build-runner
-
-    - name: Upload with runner
-      run: |
-        # Deliberately don't use TEST_MODE here. This is specifically testing
-        # the compatibility with the API.
-        runner/dist/codeql-runner-linux upload --sarif-file src/testdata/empty-sarif.sarif --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}

.github/workflows/pr-checks.yml

@@ -1,6 +1,6 @@
 name: "PR checks"
 
-on: [push, pull_request]
+on: [pull_request]
 
 jobs:
   lint-js:

.github/workflows/runner.yml

@@ -0,0 +1,26 @@
+name: "CodeQL runner"
+
+on: [pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Build runner
+      run: |
+        cd runner
+        npm install
+        npm run build-runner
+
+    - name: Upload with runner
+      run: |
+        runner/dist/codeql-runner-linux upload \
+          --sarif-file src/testdata/empty-sarif.sarif \
+          --repository $GITHUB_REPOSITORY \
+          --commit $GITHUB_SHA \
+          --ref $GITHUB_REF \
+          --github-url $GITHUB_SERVER_URL \
+          --github-auth ${{ github.token }}

lib/codeql.js

@@ -359,7 +359,7 @@ function getCodeQLForCmd(cmd) {
             return JSON.parse(output);
         },
         async databaseAnalyze(databasePath, sarifFile, querySuite, memoryFlag, addSnippetsFlag, threadsFlag) {
-            await new toolrunnner.ToolRunner(cmd, [
+            await toolrunner_error_catcher_1.toolrunnerErrorCatcher(cmd, [
                 "database",
                 "analyze",
                 memoryFlag,
@@ -370,7 +370,7 @@ function getCodeQLForCmd(cmd) {
                 addSnippetsFlag,
                 ...getExtraOptionsFromEnv(["database", "analyze"]),
                 querySuite,
-            ]).exec();
+            ]);
         },
     };
 }

lib/error-matcher.js

@@ -11,6 +11,17 @@ exports.namedMatchersForTesting = {
         message: "No code found during the build. Please see:\n" +
             "https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/troubleshooting-code-scanning#no-code-found-during-the-build",
     },
+    queryTimeout: {
+        exitCode: 33,
+        message: "One or more query evaluations timed out. Please see:\n" +
+            "Not sure if there's a good place to currently send people?",
+    },
+    outOfMemory: {
+        exitCode: 99,
+        outputRegex: new RegExp("Operation ran out of memory\\."),
+        message: "The CodeQL CLI ran out of memory. Please see:\n" +
+            "https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/troubleshooting-code-scanning#error-out-of-disk-or-out-of-memory",
+    },
 };
 // we collapse the matches into an array for use in execErrorCatcher
 exports.errorMatchers = Object.values(exports.namedMatchersForTesting);

lib/error-matcher.js.map

@@ -1 +1 @@
-{"version":3,"file":"error-matcher.js","sourceRoot":"","sources":["../src/error-matcher.ts"],"names":[],"mappings":";;AAQA,qCAAqC;AACxB,QAAA,uBAAuB,GAAoC;IACtE;;MAEE;IACF,iBAAiB,EAAE;QACjB,QAAQ,EAAE,EAAE;QACZ,WAAW,EAAE,IAAI,MAAM,CAAC,2CAA2C,CAAC;QACpE,OAAO,EACL,+CAA+C;YAC/C,yJAAyJ;KAC5J;CACF,CAAC;AAEF,oEAAoE;AACvD,QAAA,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,+BAAuB,CAAC,CAAC"}
+{"version":3,"file":"error-matcher.js","sourceRoot":"","sources":["../src/error-matcher.ts"],"names":[],"mappings":";;AAQA,qCAAqC;AACxB,QAAA,uBAAuB,GAAoC;IACtE;;MAEE;IACF,iBAAiB,EAAE;QACjB,QAAQ,EAAE,EAAE;QACZ,WAAW,EAAE,IAAI,MAAM,CAAC,2CAA2C,CAAC;QACpE,OAAO,EACL,+CAA+C;YAC/C,yJAAyJ;KAC5J;IAED,YAAY,EAAE;QACZ,QAAQ,EAAE,EAAE;QACZ,OAAO,EACL,wDAAwD;YACxD,4DAA4D;KAC/D;IAED,WAAW,EAAE;QACX,QAAQ,EAAE,EAAE;QACZ,WAAW,EAAE,IAAI,MAAM,CAAC,gCAAgC,CAAC;QACzD,OAAO,EACL,iDAAiD;YACjD,6JAA6J;KAChK;CACF,CAAC;AAEF,oEAAoE;AACvD,QAAA,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,+BAAuB,CAAC,CAAC"}

lib/toolrunner-error-catcher.test.js

@@ -23,6 +23,31 @@ ava_1.default("matchers are never applied if non-error exit", async (t) => {
     t.deepEqual(await exec.exec("node", testArgs), 0);
     t.deepEqual(await toolrunner_error_catcher_1.toolrunnerErrorCatcher("node", testArgs, matchers), 0);
 });
+ava_1.default("exit code matching is correctly handled", async (t) => {
+    const testArgs = buildDummyArgs("foo bar\\nblort qux", "foo bar\\nblort qux", "", 42);
+    const nonMatchers = [
+        {
+            exitCode: 123,
+            outputRegex: new RegExp("will not match"),
+            message: "custom message!",
+        },
+    ];
+    await t.throwsAsync(toolrunner_error_catcher_1.toolrunnerErrorCatcher("node", testArgs, nonMatchers), {
+        instanceOf: Error,
+        message: "The process 'node' failed with exit code 42",
+    });
+    const matchers = [
+        {
+            exitCode: 42,
+            outputRegex: new RegExp("will not match"),
+            message: "custom message!",
+        },
+    ];
+    await t.throwsAsync(toolrunner_error_catcher_1.toolrunnerErrorCatcher("node", testArgs, matchers), {
+        instanceOf: Error,
+        message: "custom message!",
+    });
+});
 ava_1.default("regex matchers are applied to stdout for non-zero exit code", async (t) => {
     const testArgs = buildDummyArgs("foo bar\\nblort qux", "", "", 1);
     const matchers = [

src/codeql.ts

@@ -565,7 +565,7 @@ function getCodeQLForCmd(cmd: string): CodeQL {
       addSnippetsFlag: string,
       threadsFlag: string
     ) {
-      await new toolrunnner.ToolRunner(cmd, [
+      await toolrunnerErrorCatcher(cmd, [
         "database",
         "analyze",
         memoryFlag,
@@ -576,7 +576,7 @@ function getCodeQLForCmd(cmd: string): CodeQL {
         addSnippetsFlag,
         ...getExtraOptionsFromEnv(["database", "analyze"]),
         querySuite,
-      ]).exec();
+      ]);
     },
   };
 }

src/error-matcher.ts

@@ -18,6 +18,21 @@ export const namedMatchersForTesting: { [key: string]: ErrorMatcher } = {
       "No code found during the build. Please see:\n" +
       "https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/troubleshooting-code-scanning#no-code-found-during-the-build",
   },
+
+  queryTimeout: {
+    exitCode: 33,
+    message:
+      "One or more query evaluations timed out. Please see:\n" +
+      "Not sure if there's a good place to currently send people?", // TODO find where we should send people for this error.
+  },
+
+  outOfMemory: {
+    exitCode: 99,
+    outputRegex: new RegExp("Operation ran out of memory\\."),
+    message:
+      "The CodeQL CLI ran out of memory. Please see:\n" +
+      "https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/troubleshooting-code-scanning#error-out-of-disk-or-out-of-memory",
+  },
 };
 
 // we collapse the matches into an array for use in execErrorCatcher

src/toolrunner-error-catcher.test.ts

@@ -24,6 +24,41 @@ test("matchers are never applied if non-error exit", async (t) => {
   t.deepEqual(await toolrunnerErrorCatcher("node", testArgs, matchers), 0);
 });
 
+test("exit code matching is correctly handled", async (t) => {
+  const testArgs = buildDummyArgs(
+    "foo bar\\nblort qux",
+    "foo bar\\nblort qux",
+    "",
+    42
+  );
+
+  const nonMatchers: ErrorMatcher[] = [
+    {
+      exitCode: 123,
+      outputRegex: new RegExp("will not match"),
+      message: "custom message!",
+    },
+  ];
+
+  await t.throwsAsync(toolrunnerErrorCatcher("node", testArgs, nonMatchers), {
+    instanceOf: Error,
+    message: "The process 'node' failed with exit code 42",
+  });
+
+  const matchers: ErrorMatcher[] = [
+    {
+      exitCode: 42,
+      outputRegex: new RegExp("will not match"),
+      message: "custom message!",
+    },
+  ];
+
+  await t.throwsAsync(toolrunnerErrorCatcher("node", testArgs, matchers), {
+    instanceOf: Error,
+    message: "custom message!",
+  });
+});
+
 test("regex matchers are applied to stdout for non-zero exit code", async (t) => {
   const testArgs = buildDummyArgs("foo bar\\nblort qux", "", "", 1);