Make deps directory persistent over upgrades (#7801)

* Use pip install --user if venv not active

* Set PYTHONUSERBASE to deps directory, when installing with --user
  option.
* Reset --prefix option to workaround incompatability when installing
  with --user option. This requires pip version 8.0.0 or greater.
* Require pip version 8.0.3.
* Do not delete deps directory on home assistant upgrade.
* Fix local lib mount and check package exist.

* Update and add tests

* Fix upgrade from before version 0.46

* Extract function to get user site

* Add function(s) to package util to get user site.
* Use async subprocess for one of the functions to get user site.
* Add function to package util to check if virtual environment is
  active.
* Add and update tests.

* Update version for last removal of deps dir

* Address comments

* Rewrite package util tests with pytest

* Rewrite all existing unittest class based tests for package util as
  test functions, and capitalize pytest fixtures.
* Add test for installing with target inside venv.
This commit is contained in:
Martin Hjelmare authored and Paulus Schoutsen committed 2017-07-13 19:26:21 -07:00
1 parent 5581c6295e
commit ba019c799a
11 files changed
+364 -167

No files matched your search

+197 -119
View File
@@ -1,11 +1,13 @@
"""Test Home Assistant package util methods."""
import asyncio
import logging
import os
import pkg_resources
import unittest
import sys
from subprocess import PIPE
from distutils.sysconfig import get_python_lib
from unittest.mock import call, patch, Mock
from unittest.mock import MagicMock, call, patch
import pkg_resources
import pytest
import homeassistant.util.package as package
@@ -18,124 +20,200 @@ TEST_ZIP_REQ = 'file://{}#{}' \
.format(os.path.join(RESOURCE_DIR, 'pyhelloworld3.zip'), TEST_NEW_REQ)
@patch('homeassistant.util.package.Popen')
@patch('homeassistant.util.package.check_package_exists')
class TestPackageUtilInstallPackage(unittest.TestCase):
"""Test for homeassistant.util.package module."""
def setUp(self):
"""Setup the tests."""
self.mock_process = Mock()
self.mock_process.communicate.return_value = (b'message', b'error')
self.mock_process.returncode = 0
def test_install_existing_package(self, mock_exists, mock_popen):
"""Test an install attempt on an existing package."""
mock_popen.return_value = self.mock_process
mock_exists.return_value = True
self.assertTrue(package.install_package(TEST_EXIST_REQ))
self.assertEqual(mock_exists.call_count, 1)
self.assertEqual(mock_exists.call_args, call(TEST_EXIST_REQ, None))
self.assertEqual(self.mock_process.communicate.call_count, 0)
@patch('homeassistant.util.package.sys')
def test_install(self, mock_sys, mock_exists, mock_popen):
"""Test an install attempt on a package that doesn't exist."""
mock_exists.return_value = False
mock_popen.return_value = self.mock_process
self.assertTrue(package.install_package(TEST_NEW_REQ, False))
self.assertEqual(mock_exists.call_count, 1)
self.assertEqual(self.mock_process.communicate.call_count, 1)
self.assertEqual(mock_popen.call_count, 1)
self.assertEqual(
mock_popen.call_args,
call([
mock_sys.executable, '-m', 'pip', 'install', '--quiet',
TEST_NEW_REQ
], stdin=PIPE, stdout=PIPE, stderr=PIPE)
)
@patch('homeassistant.util.package.sys')
def test_install_upgrade(self, mock_sys, mock_exists, mock_popen):
"""Test an upgrade attempt on a package."""
mock_exists.return_value = False
mock_popen.return_value = self.mock_process
self.assertTrue(package.install_package(TEST_NEW_REQ))
self.assertEqual(mock_exists.call_count, 1)
self.assertEqual(self.mock_process.communicate.call_count, 1)
self.assertEqual(mock_popen.call_count, 1)
self.assertEqual(
mock_popen.call_args,
call([
mock_sys.executable, '-m', 'pip', 'install', '--quiet',
TEST_NEW_REQ, '--upgrade'
], stdin=PIPE, stdout=PIPE, stderr=PIPE)
)
@patch('homeassistant.util.package.sys')
def test_install_target(self, mock_sys, mock_exists, mock_popen):
"""Test an install with a target."""
target = 'target_folder'
mock_exists.return_value = False
mock_popen.return_value = self.mock_process
self.assertTrue(
package.install_package(TEST_NEW_REQ, False, target=target)
)
self.assertEqual(mock_exists.call_count, 1)
self.assertEqual(self.mock_process.communicate.call_count, 1)
self.assertEqual(mock_popen.call_count, 1)
self.assertEqual(
mock_popen.call_args,
call([
mock_sys.executable, '-m', 'pip', 'install', '--quiet',
TEST_NEW_REQ, '--target', os.path.abspath(target)
], stdin=PIPE, stdout=PIPE, stderr=PIPE)
)
@patch('homeassistant.util.package._LOGGER')
@patch('homeassistant.util.package.sys')
def test_install_error(self, mock_sys, mock_logger, mock_exists,
mock_popen):
"""Test an install with a target."""
mock_exists.return_value = False
mock_popen.return_value = self.mock_process
self.mock_process.returncode = 1
self.assertFalse(package.install_package(TEST_NEW_REQ))
self.assertEqual(mock_logger.error.call_count, 1)
@pytest.fixture
def mock_sys():
"""Mock sys."""
with patch('homeassistant.util.package.sys', spec=object) as sys_mock:
sys_mock.executable = 'python3'
yield sys_mock
class TestPackageUtilCheckPackageExists(unittest.TestCase):
"""Test for homeassistant.util.package module."""
@pytest.fixture
def mock_exists():
"""Mock check_package_exists."""
with patch('homeassistant.util.package.check_package_exists') as mock:
mock.return_value = False
yield mock
def test_check_package_global(self):
"""Test for a globally-installed package."""
installed_package = list(pkg_resources.working_set)[0].project_name
self.assertTrue(package.check_package_exists(installed_package, None))
@pytest.fixture
def deps_dir():
"""Return path to deps directory."""
return os.path.abspath('/deps_dir')
def test_check_package_local(self):
"""Test for a locally-installed package."""
lib_dir = get_python_lib()
installed_package = list(pkg_resources.working_set)[0].project_name
self.assertTrue(
package.check_package_exists(installed_package, lib_dir)
)
@pytest.fixture
def lib_dir(deps_dir):
"""Return path to lib directory."""
return os.path.join(deps_dir, 'lib_dir')
def test_check_package_zip(self):
"""Test for an installed zip package."""
self.assertFalse(package.check_package_exists(TEST_ZIP_REQ, None))
@pytest.fixture
def mock_popen(lib_dir):
"""Return a Popen mock."""
with patch('homeassistant.util.package.Popen') as popen_mock:
popen_mock.return_value.communicate.return_value = (
bytes(lib_dir, 'utf-8'), b'error')
popen_mock.return_value.returncode = 0
yield popen_mock
@pytest.fixture
def mock_env_copy():
"""Mock os.environ.copy."""
with patch('homeassistant.util.package.os.environ.copy') as env_copy:
env_copy.return_value = {}
yield env_copy
@asyncio.coroutine
def mock_async_subprocess():
"""Return an async Popen mock."""
async_popen = MagicMock()
@asyncio.coroutine
def communicate(input=None):
"""Communicate mock."""
stdout = bytes('/deps_dir/lib_dir', 'utf-8')
return (stdout, None)
async_popen.communicate = communicate
return async_popen
def test_install_existing_package(mock_exists, mock_popen):
"""Test an install attempt on an existing package."""
mock_exists.return_value = True
assert package.install_package(TEST_EXIST_REQ)
assert mock_exists.call_count == 1
assert mock_exists.call_args == call(TEST_EXIST_REQ)
assert mock_popen.return_value.communicate.call_count == 0
def test_install(mock_sys, mock_exists, mock_popen, mock_env_copy):
"""Test an install attempt on a package that doesn't exist."""
env = mock_env_copy()
assert package.install_package(TEST_NEW_REQ, False)
assert mock_exists.call_count == 1
assert mock_popen.call_count == 1
assert (
mock_popen.call_args ==
call([
mock_sys.executable, '-m', 'pip', 'install', '--quiet',
TEST_NEW_REQ
], stdin=PIPE, stdout=PIPE, stderr=PIPE, env=env)
)
assert mock_popen.return_value.communicate.call_count == 1
def test_install_upgrade(mock_sys, mock_exists, mock_popen, mock_env_copy):
"""Test an upgrade attempt on a package."""
env = mock_env_copy()
assert package.install_package(TEST_NEW_REQ)
assert mock_exists.call_count == 1
assert mock_popen.call_count == 1
assert (
mock_popen.call_args ==
call([
mock_sys.executable, '-m', 'pip', 'install', '--quiet',
TEST_NEW_REQ, '--upgrade'
], stdin=PIPE, stdout=PIPE, stderr=PIPE, env=env)
)
assert mock_popen.return_value.communicate.call_count == 1
def test_install_target(mock_sys, mock_exists, mock_popen, mock_env_copy):
"""Test an install with a target."""
target = 'target_folder'
env = mock_env_copy()
env['PYTHONUSERBASE'] = os.path.abspath(target)
mock_sys.platform = 'linux'
args = [
mock_sys.executable, '-m', 'pip', 'install', '--quiet',
TEST_NEW_REQ, '--user', '--prefix=']
assert package.install_package(TEST_NEW_REQ, False, target=target)
assert mock_exists.call_count == 1
assert mock_popen.call_count == 1
assert (
mock_popen.call_args ==
call(args, stdin=PIPE, stdout=PIPE, stderr=PIPE, env=env)
)
assert mock_popen.return_value.communicate.call_count == 1
def test_install_target_venv(mock_sys, mock_exists, mock_popen, mock_env_copy):
"""Test an install with a target in a virtual environment."""
target = 'target_folder'
mock_sys.real_prefix = '/usr'
with pytest.raises(AssertionError):
package.install_package(TEST_NEW_REQ, False, target=target)
def test_install_error(caplog, mock_sys, mock_exists, mock_popen):
"""Test an install with a target."""
caplog.set_level(logging.WARNING)
mock_popen.return_value.returncode = 1
assert not package.install_package(TEST_NEW_REQ)
assert len(caplog.records) == 1
for record in caplog.records:
assert record.levelname == 'ERROR'
def test_install_constraint(mock_sys, mock_exists, mock_popen, mock_env_copy):
"""Test install with constraint file on not installed package."""
env = mock_env_copy()
constraints = 'constraints_file.txt'
assert package.install_package(
TEST_NEW_REQ, False, constraints=constraints)
assert mock_exists.call_count == 1
assert mock_popen.call_count == 1
assert (
mock_popen.call_args ==
call([
mock_sys.executable, '-m', 'pip', 'install', '--quiet',
TEST_NEW_REQ, '--constraint', constraints
], stdin=PIPE, stdout=PIPE, stderr=PIPE, env=env)
)
assert mock_popen.return_value.communicate.call_count == 1
def test_check_package_global():
"""Test for an installed package."""
installed_package = list(pkg_resources.working_set)[0].project_name
assert package.check_package_exists(installed_package)
def test_check_package_zip():
"""Test for an installed zip package."""
assert not package.check_package_exists(TEST_ZIP_REQ)
def test_get_user_site(deps_dir, lib_dir, mock_popen, mock_env_copy):
"""Test get user site directory."""
env = mock_env_copy()
env['PYTHONUSERBASE'] = os.path.abspath(deps_dir)
args = [sys.executable, '-m', 'site', '--user-site']
ret = package.get_user_site(deps_dir)
assert mock_popen.call_count == 1
assert mock_popen.call_args == call(
args, stdin=PIPE, stdout=PIPE, stderr=PIPE, env=env)
assert ret == lib_dir
@asyncio.coroutine
def test_async_get_user_site(hass, mock_env_copy):
"""Test async get user site directory."""
deps_dir = '/deps_dir'
env = mock_env_copy()
env['PYTHONUSERBASE'] = os.path.abspath(deps_dir)
args = [sys.executable, '-m', 'site', '--user-site']
with patch('homeassistant.util.package.asyncio.create_subprocess_exec',
return_value=mock_async_subprocess()) as popen_mock:
ret = yield from package.async_get_user_site(deps_dir, hass.loop)
assert popen_mock.call_count == 1
assert popen_mock.call_args == call(
*args, loop=hass.loop, stdin=asyncio.subprocess.PIPE,
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL,
env=env)
assert ret == os.path.join(deps_dir, 'lib_dir')