Autobuild: Prefix invocations with CODEQL_RUNNER

.github/workflows/pr-checks.yml

@@ -1,354 +1,15 @@
 name: PR Checks (Basic Checks and Runner)
 
 on:
-  push:
-    branches: [main, v1]
-  pull_request:
-    # Run checks on reopened draft PRs to support triggering PR checks on draft PRs that were opened
-    # by other workflows.
-    types: [opened, synchronize, reopened, ready_for_review]
   workflow_dispatch:
 
 jobs:
-  lint-js:
-    name: Lint
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v2
-      - name: Run Lint
-        run: npm run-script lint
-
-  check-js:
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v2
-      - name: Check generated JS
-        run: .github/workflows/script/check-js.sh
-
-  check-node-modules:
-    name: Check modules up to date
-    runs-on: macos-latest
-
-    steps:
-      - uses: actions/checkout@v2
-      - name: Check node modules up to date
-        run: .github/workflows/script/check-node-modules.sh
-
-  verify-pr-checks:
-    name: Verify PR checks up to date
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.8
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install ruamel.yaml
-      - name: Verify PR checks up to date
-        run: .github/workflows/script/verify-pr-checks.sh
-
-  npm-test:
-    name: Unit Test
-    needs: [check-js, check-node-modules]
-    strategy:
-      matrix:
-        os: [ubuntu-latest, macos-latest]
-    runs-on: ${{ matrix.os }}
-
-    steps:
-      - uses: actions/checkout@v2
-      - name: npm run-script test
-        run: npm run-script test
-
-  runner-analyze-javascript-ubuntu:
-    name: Runner ubuntu JS analyze
-    needs: [check-js, check-node-modules]
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Build runner
-        run: |
-          cd runner
-          npm install
-          npm run build-runner
-
-      - name: Run init
-        run: |
-          # Pass --config-file here, but not for other jobs in this workflow.
-          # This means we're testing the config file parsing in the runner
-          # but not slowing down all jobs unnecessarily as it doesn't add much
-          # testing the parsing on different operating systems and languages.
-          runner/dist/codeql-runner-linux init --repository $GITHUB_REPOSITORY --languages javascript --config-file ./.github/codeql/codeql-config.yml --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
-
-      - name: Run analyze
-        run: |
-          runner/dist/codeql-runner-linux analyze --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
-        env:
-          TEST_MODE: true
-
-  runner-analyze-javascript-windows:
-    name: Runner windows JS analyze
-    needs: [check-js, check-node-modules]
-    runs-on: windows-latest
-
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Build runner
-        run: |
-          cd runner
-          npm install
-          npm run build-runner
-
-      - name: Run init
-        run: |
-          runner/dist/codeql-runner-win.exe init --repository $Env:GITHUB_REPOSITORY --languages javascript --github-url $Env:GITHUB_SERVER_URL --github-auth ${{ github.token }}
-
-      - name: Run analyze
-        run: |
-          runner/dist/codeql-runner-win.exe analyze --repository $Env:GITHUB_REPOSITORY --commit $Env:GITHUB_SHA --ref $Env:GITHUB_REF --github-url $Env:GITHUB_SERVER_URL --github-auth ${{ github.token }}
-        env:
-          TEST_MODE: true
-
-  runner-analyze-javascript-macos:
-    name: Runner macos JS analyze
-    needs: [check-js, check-node-modules]
-    runs-on: macos-latest
-
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Build runner
-        run: |
-          cd runner
-          npm install
-          npm run build-runner
-
-      - name: Run init
-        run: |
-          runner/dist/codeql-runner-macos init --repository $GITHUB_REPOSITORY --languages javascript --config-file ./.github/codeql/codeql-config.yml --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
-
-      - name: Run analyze
-        run: |
-          runner/dist/codeql-runner-macos analyze --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
-        env:
-          TEST_MODE: true
-
-  runner-analyze-csharp-ubuntu:
-    name: Runner ubuntu C# analyze
-    needs: [check-js, check-node-modules]
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Move codeql-action
-        shell: bash
-        run: |
-          mkdir ../action
-          mv * .github ../action/
-          mv ../action/tests/multi-language-repo/{*,.github} .
-          mv ../action/.github/workflows .github
-
-      - name: Build runner
-        run: |
-          cd ../action/runner
-          npm install
-          npm run build-runner
-      
-      - name: Initialize dotnet
-        run: dotnet restore
-
-      - name: Run init
-        run: |
-          ../action/runner/dist/codeql-runner-linux init --repository $GITHUB_REPOSITORY --languages csharp --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
-
-      - name: Build code
-        run: |
-          . ./codeql-runner/codeql-env.sh
-          $CODEQL_RUNNER dotnet build /p:UseSharedCompilation=false
-
-      - name: Run analyze
-        run: |
-          ../action/runner/dist/codeql-runner-linux analyze --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
-        env:
-          TEST_MODE: true
-
-  runner-analyze-csharp-windows:
-    name: Runner windows C# analyze
-    needs: [check-js, check-node-modules]
-    runs-on: windows-latest
-
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Move codeql-action
-        shell: bash
-        run: |
-          mkdir ../action
-          mv * .github ../action/
-          mv ../action/tests/multi-language-repo/{*,.github} .
-          mv ../action/.github/workflows .github
-
-      - name: Build runner
-        run: |
-          cd ../action/runner
-          npm install
-          npm run build-runner
-      
-      - name: Initialize dotnet
-        run: dotnet restore
-
-      - name: Run init
-        run: |
-          ../action/runner/dist/codeql-runner-win.exe init --repository $Env:GITHUB_REPOSITORY --languages csharp --github-url $Env:GITHUB_SERVER_URL --github-auth ${{ github.token }}
-
-      - name: Build code
-        shell: powershell
-        run: |
-          cat ./codeql-runner/codeql-env.sh | Invoke-Expression
-          $Env:CODEQL_EXTRACTOR_CSHARP_ROOT = "" # Unset an environment variable to make sure the tracer resists this
-          & $Env:CODEQL_RUNNER dotnet build /p:UseSharedCompilation=false
-
-      - name: Upload tracer logs
-        uses: actions/upload-artifact@v2
-        with:
-          name: tracer-logs
-          path: ./codeql-runner/compound-build-tracer.log
-
-      - name: Run analyze
-        run: |
-          ../action/runner/dist/codeql-runner-win.exe analyze --repository $Env:GITHUB_REPOSITORY --commit $Env:GITHUB_SHA --ref $Env:GITHUB_REF --github-url $Env:GITHUB_SERVER_URL --github-auth ${{ github.token }}
-        env:
-          TEST_MODE: true
-
-  runner-analyze-csharp-macos:
-    name: Runner macos C# analyze
-    needs: [check-js, check-node-modules]
-    runs-on: macos-latest
-
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Move codeql-action
-        shell: bash
-        run: |
-          mkdir ../action
-          mv * .github ../action/
-          mv ../action/tests/multi-language-repo/{*,.github} .
-          mv ../action/.github/workflows .github
-
-      - name: Build runner
-        run: |
-          cd ../action/runner
-          npm install
-          npm run build-runner
-
-      - name: Initialize dotnet
-        run: dotnet restore
-
-      - name: Run init
-        run: |
-          ../action/runner/dist/codeql-runner-macos init --repository $GITHUB_REPOSITORY --languages csharp --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
-
-      - name: Build code
-        shell: bash
-        run: |
-          . ./codeql-runner/codeql-env.sh
-          $CODEQL_RUNNER dotnet build /p:UseSharedCompilation=false
-
-      - name: Run analyze
-        run: |
-          ../action/runner/dist/codeql-runner-macos analyze --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
-        env:
-          TEST_MODE: true
-
-  runner-analyze-csharp-autobuild-ubuntu:
-    name: Runner ubuntu autobuild C# analyze
-    needs: [check-js, check-node-modules]
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Move codeql-action
-        shell: bash
-        run: |
-          mkdir ../action
-          mv * .github ../action/
-          mv ../action/tests/multi-language-repo/{*,.github} .
-          mv ../action/.github/workflows .github
-
-      - name: Build runner
-        run: |
-          cd ../action/runner
-          npm install
-          npm run build-runner
-
-      - name: Run init
-        run: |
-          ../action/runner/dist/codeql-runner-linux init --repository $GITHUB_REPOSITORY --languages csharp --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
-
-      - name: Build code
-        run: |
-          ../action/runner/dist/codeql-runner-linux autobuild
-
-      - name: Run analyze
-        run: |
-          ../action/runner/dist/codeql-runner-linux analyze --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
-        env:
-          TEST_MODE: true
-
-  runner-analyze-csharp-autobuild-windows:
-    name: Runner windows autobuild C# analyze
-    needs: [check-js, check-node-modules]
-    runs-on: windows-latest
-
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Move codeql-action
-        shell: bash
-        run: |
-          mkdir ../action
-          mv * .github ../action/
-          mv ../action/tests/multi-language-repo/{*,.github} .
-          mv ../action/.github/workflows .github
-
-      - name: Build runner
-        run: |
-          cd ../action/runner
-          npm install
-          npm run build-runner
-
-      - name: Run init
-        run: |
-          ../action/runner/dist/codeql-runner-win.exe init --repository $Env:GITHUB_REPOSITORY --languages csharp --github-url $Env:GITHUB_SERVER_URL --github-auth ${{ github.token }}
-
-      - name: Build code
-        shell: powershell
-        run: |
-          ../action/runner/dist/codeql-runner-win.exe autobuild
-
-      - name: Run analyze
-        run: |
-          ../action/runner/dist/codeql-runner-win.exe analyze --repository $Env:GITHUB_REPOSITORY --commit $Env:GITHUB_SHA --ref $Env:GITHUB_REF --github-url $Env:GITHUB_SERVER_URL --github-auth ${{ github.token }}
-        env:
-          TEST_MODE: true
-
   runner-analyze-csharp-autobuild-macos:
     name: Runner macos autobuild C# analyze
-    needs: [check-js, check-node-modules]
     runs-on: macos-latest
 
+    env:
+      ACTIONS_RUNNER_DEBUG: 1
     steps:
       - uses: actions/checkout@v2
 
@@ -366,78 +27,43 @@ jobs:
           npm install
           npm run build-runner
 
+      - name: Initialize dotnet
+        run: dotnet restore
+
       - name: Run init
         run: |
           ../action/runner/dist/codeql-runner-macos init --repository $GITHUB_REPOSITORY --languages csharp --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
 
+      - name: Create extractor directory
+        run: |
+          mkdir -p /Users/runner/work/codeql-action/codeql-action/codeql-runner/codeql_databases/csharp/log
+          echo ""
+
+      - name: Check env
+        run: env
+
       - name: Build code
         shell: bash
         run: |
+          export CODEQL_EXTRACTOR_CSHARP_LOG_DIR="/Users/runner/work/codeql-action/codeql-action/codeql-runner/codeql_databases/csharp/log"
+          mkdir -p "$CODEQL_EXTRACTOR_CSHARP_LOG_DIR"
+          touch "$CODEQL_EXTRACTOR_CSHARP_LOG_DIR/csharp.log"
           ../action/runner/dist/codeql-runner-macos autobuild
 
+      - name: Check env
+        if: always()
+        run: env
+
+      - name: Upload
+        if: always()
+        uses: actions/upload-artifact@v2
+        with:
+          path: /Users/runner/work/codeql-action/codeql-action/codeql-runner/codeql_databases/csharp
+
       - name: Run analyze
         run: |
           ../action/runner/dist/codeql-runner-macos analyze --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
         env:
           TEST_MODE: true
 
-  runner-upload-sarif:
-    name: Runner upload sarif
-    needs: [check-js, check-node-modules]
-    runs-on: ubuntu-latest
-
-    if: ${{ github.event_name != 'pull_request' || github.event.pull_request.base.repo.id == github.event.pull_request.head.repo.id }}
-
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Build runner
-        run: |
-          cd runner
-          npm install
-          npm run build-runner
-
-      - name: Upload with runner
-        run: |
-          # Deliberately don't use TEST_MODE here. This is specifically testing
-          # the compatibility with the API.
-          runner/dist/codeql-runner-linux upload --sarif-file src/testdata/empty-sarif.sarif --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
-
-  runner-extractor-ram-threads-options:
-    name: Runner ubuntu extractor RAM and threads options
-    needs: [check-js, check-node-modules]
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Build runner
-        run: |
-          cd runner
-          npm install
-          npm run build-runner
-
-      - name: Run init
-        run: |
-          runner/dist/codeql-runner-linux init --ram=230 --threads=1 --repository $GITHUB_REPOSITORY --languages java --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
-
-      - name: Assert Results
-        shell: bash
-        run: |
-          . ./codeql-runner/codeql-env.sh
-          if [ "${CODEQL_RAM}" != "230" ]; then
-            echo "CODEQL_RAM is '${CODEQL_RAM}' instead of 230"
-            exit 1
-          fi
-          if [ "${CODEQL_EXTRACTOR_JAVA_RAM}" != "230" ]; then
-            echo "CODEQL_EXTRACTOR_JAVA_RAM is '${CODEQL_EXTRACTOR_JAVA_RAM}' instead of 230"
-            exit 1
-          fi
-          if [ "${CODEQL_THREADS}" != "1" ]; then
-            echo "CODEQL_THREADS is '${CODEQL_THREADS}' instead of 1"
-            exit 1
-          fi
-          if [ "${CODEQL_EXTRACTOR_JAVA_THREADS}" != "1" ]; then
-            echo "CODEQL_EXTRACTOR_JAVA_THREADS is '${CODEQL_EXTRACTOR_JAVA_THREADS}' instead of 1"
-            exit 1
-          fi
+  

src/codeql.ts

@@ -732,7 +732,17 @@ async function getCodeQLForCmd(
         "-Dmaven.wagon.http.pool=false",
       ].join(" ");
 
-      await runTool(autobuildCmd);
+      const runnerExecutable = process.env["CODEQL_RUNNER"] || "";
+      // On Mac, prefixing with the runner executable is required to handle System Integrity Protection.
+      if (runnerExecutable) {
+        // Earlier steps (init) are expected to have written the runner executable path
+        // to the tracing environment, and the current step is expected to have
+        // correctly loaded that environment.
+        await runTool(runnerExecutable, [autobuildCmd]);
+      } else {
+        // Fallback in case CODEQL_RUNNER wasn't correctly set or loaded.
+        await runTool(autobuildCmd);
+      }
     },
     async extractScannedLanguage(databasePath: string, language: Language) {
       // Get extractor location