#!/usr/bin/env python3 import sys import os import subprocess from tempfile import mkdtemp from typing import Optional import extractor_version def _check_call(command): print('+ {}'.format(' '.join(command)), flush=True) subprocess.check_call(command, stdin=subprocess.DEVNULL) def _check_output(command): print('+ {}'.format(' '.join(command)), flush=True) out = subprocess.check_output(command, stdin=subprocess.DEVNULL) print(out, flush=True) sys.stderr.flush() return out def install_packages_with_poetry(): command = [sys.executable, '-m', 'poetry'] if sys.platform.startswith('win32'): # In windows the default path were the deps are installed gets wiped out between steps, # so we have to set it up to a folder that will be kept os.environ['POETRY_VIRTUALENVS_PATH'] = os.path.join(os.environ['RUNNER_WORKSPACE'], 'virtualenvs') try: _check_call(command + ['install', '--no-root']) except subprocess.CalledProcessError: sys.exit('package installation with poetry failed, see error above') # poetry is super annoying with `poetry run`, since it will put lots of output on # STDOUT if the current global python interpreter is not matching the one in the # virtualenv for the package, which was the case for using poetry for Python 2 when # default system interpreter was Python 3 :/ poetry_out = _check_output(command + ['run', 'which', 'python']) python_executable_path = poetry_out.decode('utf-8').splitlines()[-1] if sys.platform.startswith('win32'): # Poetry produces a path that starts by /d instead of D:\ and Windows doesn't like that way of specifying the drive letter. # We completely remove it because it is not needed as everything is in the same drive (We are installing the dependencies in the RUNNER_WORKSPACE) python_executable_path = python_executable_path[2:] return python_executable_path def install_packages_with_pipenv(): command = [sys.executable, '-m', 'pipenv'] if sys.platform.startswith('win32'): # In windows the default path were the deps are installed gets wiped out between steps, # so we have to set it up to a folder that will be kept os.environ['WORKON_HOME'] = os.path.join(os.environ['RUNNER_WORKSPACE'], 'virtualenvs') try: _check_call(command + ['install', '--keep-outdated', '--ignore-pipfile']) except subprocess.CalledProcessError: sys.exit('package installation with pipenv failed, see error above') pipenv_out = _check_output(command + ['run', 'which', 'python']) python_executable_path = pipenv_out.decode('utf-8').splitlines()[-1] if sys.platform.startswith('win32'): # Pipenv produces a path that starts by /d instead of D:\ and Windows doesn't like that way of specifying the drive letter. # We completely remove it because it is not needed as everything is in the same drive (We are installing the dependencies in the RUNNER_WORKSPACE) python_executable_path = python_executable_path[2:] return python_executable_path def _create_venv(version: int): # create temporary directory ... that just lives "forever" venv_path = os.path.join(os.environ['RUNNER_WORKSPACE'], 'codeql-action-python-autoinstall') print ("Creating venv in " + venv_path, flush = True) # virtualenv is a bit nicer for setting up virtual environment, since it will provide # up-to-date versions of pip/setuptools/wheel which basic `python3 -m venv venv` won't if sys.platform.startswith('win32'): if version == 2: _check_call(['py', '-2', '-m', 'virtualenv', venv_path]) elif version == 3: _check_call(['py', '-3', '-m', 'virtualenv', venv_path]) else: if version == 2: _check_call(['python2', '-m', 'virtualenv', venv_path]) elif version == 3: _check_call(['python3', '-m', 'virtualenv', venv_path]) return venv_path def install_requirements_txt_packages(version: int): venv_path = _create_venv(version) venv_pip = os.path.join(venv_path, 'bin', 'pip') venv_python = os.path.join(venv_path, 'bin', 'python') if sys.platform.startswith('win32'): venv_pip = os.path.join(venv_path, 'Scripts', 'pip') venv_python = os.path.join(venv_path, 'Scripts', 'python') try: _check_call([venv_pip, 'install', '-r', 'requirements.txt']) except subprocess.CalledProcessError: sys.exit('package installation with `pip install -r requirements.txt` failed, see error above') return venv_python def install_with_setup_py(version: int): venv_path = _create_venv(version) venv_pip = os.path.join(venv_path, 'bin', 'pip') venv_python = os.path.join(venv_path, 'bin', 'python') if sys.platform.startswith('win32'): venv_pip = os.path.join(venv_path, 'Scripts', 'pip') venv_python = os.path.join(venv_path, 'Scripts', 'python') try: # We have to choose between `python setup.py develop` and `pip install -e .`. # Modern projects use `pip install -e .` and I wasn't able to see any downsides # to doing so. However, `python setup.py develop` has some downsides -- from # https://stackoverflow.com/a/19048754 : # > Note that it is highly recommended to use pip install . (install) and pip # > install -e . (developer install) to install packages, as invoking setup.py # > directly will do the wrong things for many dependencies, such as pull # > prereleases and incompatible package versions, or make the package hard to # > uninstall with pip. _check_call([venv_pip, 'install', '-e', '.']) except subprocess.CalledProcessError: sys.exit('package installation with `pip install -e .` failed, see error above') return venv_python def install_packages(codeql_base_dir) -> Optional[str]: if os.path.exists('poetry.lock'): print('Found poetry.lock, will install packages with poetry', flush=True) return install_packages_with_poetry() if os.path.exists('Pipfile') or os.path.exists('Pipfile.lock'): if os.path.exists('Pipfile.lock'): print('Found Pipfile.lock, will install packages with Pipenv', flush=True) else: print('Found Pipfile, will install packages with Pipenv', flush=True) return install_packages_with_pipenv() # get_extractor_version returns the Python version the extractor thinks this repo is using version = extractor_version.get_extractor_version(codeql_base_dir, quiet=False) if os.path.exists('requirements.txt'): print('Found requirements.txt, will install packages with pip', flush=True) return install_requirements_txt_packages(version) if os.path.exists('setup.py'): print('Found setup.py, will install package with pip in editable mode', flush=True) return install_with_setup_py(version) print("was not able to install packages automatically", flush=True) return None if __name__ == "__main__": if len(sys.argv) != 2: sys.exit('Must provide base directory for codeql tool as only argument') codeql_base_dir = sys.argv[1] python_executable_path = install_packages(codeql_base_dir) if python_executable_path is not None: # see https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable env_file = open(os.environ["GITHUB_ENV"], mode="at") print("Setting CODEQL_PYTHON={}".format(python_executable_path)) print("CODEQL_PYTHON={}".format(python_executable_path), file=env_file)