Add Jedi support for extra paths and different environment handling (#680)

Author: goanpeca

.circleci/config.yml

@@ -17,6 +17,9 @@ jobs:
       - image: "python:3.5-stretch"
     steps:
       - checkout
+      # To test Jedi environments
+      - run: python3 -m venv /tmp/pyenv
+      - run: /tmp/pyenv/bin/python -m pip install loghub
       - run: pip install -e .[all] .[test]
       - run: py.test -v test/
 

pyls/python_ls.py

@@ -202,10 +202,10 @@ def m_initialize(self, processId=None, rootUri=None, rootPath=None, initializati
 
         self.workspaces.pop(self.root_uri, None)
         self.root_uri = rootUri
-        self.workspace = Workspace(rootUri, self._endpoint)
-        self.workspaces[rootUri] = self.workspace
         self.config = config.Config(rootUri, initializationOptions or {},
                                     processId, _kwargs.get('capabilities', {}))
+        self.workspace = Workspace(rootUri, self._endpoint, self.config)
+        self.workspaces[rootUri] = self.workspace
         self._dispatchers = self._hook('pyls_dispatchers')
         self._hook('pyls_initialize')
 
@@ -355,6 +355,7 @@ def m_workspace__did_change_configuration(self, settings=None):
         self.config.update((settings or {}).get('pyls', {}))
         for workspace_uri in self.workspaces:
             workspace = self.workspaces[workspace_uri]
+            workspace.update_config(self.config)
             for doc_uri in workspace.documents:
                 self.lint(doc_uri, is_saved=False)
 
@@ -365,7 +366,7 @@ def m_workspace__did_change_workspace_folders(self, added=None, removed=None, **
 
         for added_info in added:
             added_uri = added_info['uri']
-            self.workspaces[added_uri] = Workspace(added_uri, self._endpoint)
+            self.workspaces[added_uri] = Workspace(added_uri, self._endpoint, self.config)
 
         # Migrate documents that are on the root workspace and have a better
         # match now

pyls/workspace.py

@@ -21,13 +21,17 @@ class Workspace(object):
     M_APPLY_EDIT = 'workspace/applyEdit'
     M_SHOW_MESSAGE = 'window/showMessage'
 
-    def __init__(self, root_uri, endpoint):
+    def __init__(self, root_uri, endpoint, config=None):
+        self._config = config
         self._root_uri = root_uri
         self._endpoint = endpoint
         self._root_uri_scheme = uris.urlparse(self._root_uri)[0]
         self._root_path = uris.to_fs_path(self._root_uri)
         self._docs = {}
 
+        # Cache jedi environments
+        self._environments = {}
+
         # Whilst incubating, keep rope private
         self.__rope = None
         self.__rope_config = None
@@ -77,6 +81,11 @@ def update_document(self, doc_uri, change, version=None):
         self._docs[doc_uri].apply_change(change)
         self._docs[doc_uri].version = version
 
+    def update_config(self, config):
+        self._config = config
+        for doc_uri in self.documents:
+            self.get_document(doc_uri).update_config(config)
+
     def apply_edit(self, edit):
         return self._endpoint.request(self.M_APPLY_EDIT, {'edit': edit})
 
@@ -97,17 +106,21 @@ def _create_document(self, doc_uri, source=None, version=None):
             doc_uri, source=source, version=version,
             extra_sys_path=self.source_roots(path),
             rope_project_builder=self._rope_project_builder,
+            config=self._config, workspace=self,
         )
 
 
 class Document(object):
 
-    def __init__(self, uri, source=None, version=None, local=True, extra_sys_path=None, rope_project_builder=None):
+    def __init__(self, uri, source=None, version=None, local=True, extra_sys_path=None, rope_project_builder=None,
+                 config=None, workspace=None):
         self.uri = uri
         self.version = version
         self.path = uris.to_fs_path(uri)
         self.filename = os.path.basename(self.path)
 
+        self._config = config
+        self._workspace = workspace
         self._local = local
         self._source = source
         self._extra_sys_path = extra_sys_path or []
@@ -131,6 +144,9 @@ def source(self):
                 return f.read()
         return self._source
 
+    def update_config(self, config):
+        self._config = config
+
     def apply_change(self, change):
         """Apply a change to the document."""
         text = change['text']
@@ -197,28 +213,58 @@ def word_at_position(self, position):
         return m_start[0] + m_end[-1]
 
     def jedi_names(self, all_scopes=False, definitions=True, references=False):
+        environment_path = None
+        if self._config:
+            jedi_settings = self._config.plugin_settings('jedi', document_path=self.path)
+            environment_path = jedi_settings.get('environment')
+        environment = self.get_enviroment(environment_path) if environment_path else None
+
         return jedi.api.names(
             source=self.source, path=self.path, all_scopes=all_scopes,
-            definitions=definitions, references=references
+            definitions=definitions, references=references, environment=environment,
         )
 
     def jedi_script(self, position=None):
+        extra_paths = []
+        environment_path = None
+
+        if self._config:
+            jedi_settings = self._config.plugin_settings('jedi', document_path=self.path)
+            environment_path = jedi_settings.get('environment')
+            extra_paths = jedi_settings.get('extra_paths') or []
+
+        sys_path = self.sys_path(environment_path) + extra_paths
+        environment = self.get_enviroment(environment_path) if environment_path else None
+
         kwargs = {
             'source': self.source,
             'path': self.path,
-            'sys_path': self.sys_path()
+            'sys_path': sys_path,
+            'environment': environment,
         }
+
         if position:
             kwargs['line'] = position['line'] + 1
             kwargs['column'] = _utils.clip_column(position['character'], self.lines, position['line'])
+
         return jedi.Script(**kwargs)
 
-    def sys_path(self):
+    def get_enviroment(self, environment_path=None):
+        # TODO(gatesn): #339 - make better use of jedi environments, they seem pretty powerful
+        if environment_path is None:
+            environment = jedi.api.environment.get_cached_default_environment()
+        else:
+            if environment_path in self._workspace._environments:
+                environment = self._workspace._environments[environment_path]
+            else:
+                environment = jedi.api.environment.create_environment(path=environment_path, safe=False)
+                self._workspace._environments[environment_path] = environment
+
+        return environment
+
+    def sys_path(self, environment_path=None):
         # Copy our extra sys path
         path = list(self._extra_sys_path)
-
-        # TODO(gatesn): #339 - make better use of jedi environments, they seem pretty powerful
-        environment = jedi.api.environment.get_cached_default_environment()
+        environment = self.get_enviroment(environment_path=environment_path)
         path.extend(environment.get_sys_path())
-
         return path

test/plugins/test_completion.py

@@ -1,6 +1,9 @@
 # Copyright 2017 Palantir Technologies, Inc.
 from distutils.version import LooseVersion
 import os
+import sys
+
+from test.test_utils import MockWorkspace
 import jedi
 import pytest
 
@@ -9,10 +12,13 @@
 from pyls.plugins.jedi_completion import pyls_completions as pyls_jedi_completions
 from pyls.plugins.rope_completion import pyls_completions as pyls_rope_completions
 
+
+PY2 = sys.version[0] == '2'
+LINUX = sys.platform.startswith('linux')
+CI = os.environ.get('CI')
 LOCATION = os.path.realpath(
     os.path.join(os.getcwd(), os.path.dirname(__file__))
 )
-
 DOC_URI = uris.from_fs_path(__file__)
 DOC = """import os
 print os.path.isabs("/tmp")
@@ -200,3 +206,62 @@ def test_multistatement_snippet(config):
     position = {'line': 0, 'character': len(document)}
     completions = pyls_jedi_completions(config, doc, position)
     assert completions[0]['insertText'] == 'date(${1:year}, ${2:month}, ${3:day})$0'
+
+
+def test_jedi_completion_extra_paths(config, tmpdir):
+    # Create a tempfile with some content and pass to extra_paths
+    temp_doc_content = '''
+def spam():
+    pass
+'''
+    p = tmpdir.mkdir("extra_path")
+    extra_paths = [str(p)]
+    p = p.join("foo.py")
+    p.write(temp_doc_content)
+
+    # Content of doc to test completion
+    doc_content = """import foo
+foo.s"""
+    doc = Document(DOC_URI, doc_content)
+
+    # After 'foo.s' without extra paths
+    com_position = {'line': 1, 'character': 5}
+    completions = pyls_jedi_completions(config, doc, com_position)
+    assert completions is None
+
+    # Update config extra paths
+    config.update({'plugins': {'jedi': {'extra_paths': extra_paths}}})
+    doc.update_config(config)
+
+    # After 'foo.s' with extra paths
+    com_position = {'line': 1, 'character': 5}
+    completions = pyls_jedi_completions(config, doc, com_position)
+    assert completions[0]['label'] == 'spam()'
+
+
+@pytest.mark.skipif(PY2 or not LINUX or not CI, reason="tested on linux and python 3 only")
+def test_jedi_completion_environment(config):
+    # Content of doc to test completion
+    doc_content = '''import logh
+'''
+    doc = Document(DOC_URI, doc_content, workspace=MockWorkspace())
+
+    # After 'import logh' with default environment
+    com_position = {'line': 0, 'character': 11}
+
+    assert os.path.isdir('/tmp/pyenv/')
+
+    config.update({'plugins': {'jedi': {'environment': None}}})
+    doc.update_config(config)
+    completions = pyls_jedi_completions(config, doc, com_position)
+    assert completions is None
+
+    # Update config extra environment
+    env_path = '/tmp/pyenv/bin/python'
+    config.update({'plugins': {'jedi': {'environment': env_path}}})
+    doc.update_config(config)
+
+    # After 'import logh' with new environment
+    completions = pyls_jedi_completions(config, doc, com_position)
+    assert completions[0]['label'] == 'loghub'
+    assert 'changelog generator' in completions[0]['documentation'].lower()

test/plugins/test_symbols.py

@@ -1,9 +1,19 @@
 # Copyright 2017 Palantir Technologies, Inc.
+import os
+import sys
+
+from test.test_utils import MockWorkspace
+import pytest
+
 from pyls import uris
 from pyls.plugins.symbols import pyls_document_symbols
 from pyls.lsp import SymbolKind
 from pyls.workspace import Document
 
+
+PY2 = sys.version[0] == '2'
+LINUX = sys.platform.startswith('linux')
+CI = os.environ.get('CI')
 DOC_URI = uris.from_fs_path(__file__)
 DOC = """import sys
 
@@ -21,6 +31,23 @@ def main(x):
 """
 
 
+def helper_check_symbols_all_scope(symbols):
+    # All eight symbols (import sys, a, B, __init__, x, y, main, y)
+    assert len(symbols) == 8
+
+    def sym(name):
+        return [s for s in symbols if s['name'] == name][0]
+
+    # Check we have some sane mappings to VSCode constants
+    assert sym('a')['kind'] == SymbolKind.Variable
+    assert sym('B')['kind'] == SymbolKind.Class
+    assert sym('__init__')['kind'] == SymbolKind.Function
+    assert sym('main')['kind'] == SymbolKind.Function
+
+    # Not going to get too in-depth here else we're just testing Jedi
+    assert sym('a')['location']['range']['start'] == {'line': 2, 'character': 0}
+
+
 def test_symbols(config):
     doc = Document(DOC_URI, DOC)
     config.update({'plugins': {'jedi_symbols': {'all_scopes': False}}})
@@ -49,18 +76,16 @@ def sym(name):
 def test_symbols_all_scopes(config):
     doc = Document(DOC_URI, DOC)
     symbols = pyls_document_symbols(config, doc)
+    helper_check_symbols_all_scope(symbols)
 
-    # All eight symbols (import sys, a, B, __init__, x, y, main, y)
-    assert len(symbols) == 8
-
-    def sym(name):
-        return [s for s in symbols if s['name'] == name][0]
 
-    # Check we have some sane mappings to VSCode constants
-    assert sym('a')['kind'] == SymbolKind.Variable
-    assert sym('B')['kind'] == SymbolKind.Class
-    assert sym('__init__')['kind'] == SymbolKind.Function
-    assert sym('main')['kind'] == SymbolKind.Function
+@pytest.mark.skipif(PY2 or not LINUX or not CI, reason="tested on linux and python 3 only")
+def test_symbols_all_scopes_with_jedi_environment(config):
+    doc = Document(DOC_URI, DOC, workspace=MockWorkspace())
 
-    # Not going to get too in-depth here else we're just testing Jedi
-    assert sym('a')['location']['range']['start'] == {'line': 2, 'character': 0}
+    # Update config extra environment
+    env_path = '/tmp/pyenv/bin/python'
+    config.update({'plugins': {'jedi': {'environment': env_path}}})
+    doc.update_config(config)
+    symbols = pyls_document_symbols(config, doc)
+    helper_check_symbols_all_scope(symbols)

test/test_utils.py

@@ -1,9 +1,23 @@
 # Copyright 2017 Palantir Technologies, Inc.
 import time
+import sys
+
 import mock
+
 from pyls import _utils
 
 
+class MockWorkspace(object):
+    """Mock workspace used by tests that use jedi environment."""
+
+    def __init__(self):
+        """Mock workspace used by tests that use jedi environment."""
+        self._environments = {}
+
+        # This is to avoid pyling tests of the variable not being used
+        sys.stdout.write(str(self._environments))
+
+
 def test_debounce():
     interval = 0.1
     obj = mock.Mock()

vscode-client/package.json

@@ -35,6 +35,16 @@
                     },
                     "uniqueItems": true
                 },
+                "pyls.plugins.jedi.extra_paths": {
+                    "type": "array",
+                    "default": [],
+                    "description": "Define extra paths for jedi.Script."
+                },
+                "pyls.plugins.jedi.environment": {
+                    "type": "string",
+                    "default": null,
+                    "description": "Define environment for jedi.Script and Jedi.names."
+                },
                 "pyls.plugins.jedi_completion.enabled": {
                     "type": "boolean",
                     "default": true,